From e7d8933056802bc95252a86ce482e941cd2b10d5 Mon Sep 17 00:00:00 2001 From: Wazbat Date: Sat, 15 Apr 2023 17:12:41 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Added=20support=20for=20fetching=20?= =?UTF-8?q?user=20profiles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/status.ts | 4 +- src/api/user.ts | 91 ++++++++++++++++++++++++++++++++++ src/fetch.ts | 37 ++++++++++++++ src/server.ts | 95 +++++++++++++++++++++++++++++++++--- src/strings.ts | 1 + src/types/twitterTypes.d.ts | 97 +++++++++++++++++++++++++++++++++++++ src/types/types.d.ts | 27 ++++++++++- src/user.ts | 53 ++++++++++++++++++++ test/index.test.ts | 49 +++++++++++++++++++ 9 files changed, 445 insertions(+), 9 deletions(-) create mode 100644 src/api/user.ts create mode 100644 src/user.ts diff --git a/src/api/status.ts b/src/api/status.ts index e4578e1..cc6d648 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -187,7 +187,7 @@ export const statusAPI = async ( language: string | undefined, event: FetchEvent, flags?: InputFlags -): Promise => { +): Promise => { let conversation = await fetchConversation(status, event); let tweet = conversation?.globalObjects?.tweets?.[status] || {}; @@ -242,7 +242,7 @@ export const statusAPI = async ( } /* Creating the response objects */ - const response: APIResponse = { code: 200, message: 'OK' } as APIResponse; + const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse; const apiTweet: APITweet = (await populateTweetProperties( tweet, conversation, diff --git a/src/api/user.ts b/src/api/user.ts new file mode 100644 index 0000000..bd98a8b --- /dev/null +++ b/src/api/user.ts @@ -0,0 +1,91 @@ +import { renderCard } from '../helpers/card'; +import { Constants } from '../constants'; +import { fetchConversation, fetchUser } from '../fetch'; +import { linkFixer } from '../helpers/linkFixer'; +import { handleMosaic } from '../helpers/mosaic'; +import { colorFromPalette } from '../helpers/palette'; +import { translateTweet } from '../helpers/translate'; +import { unescapeText } from '../helpers/utils'; +import { processMedia } from '../helpers/media'; + +/* This function does the heavy lifting of processing data from Twitter API + and using it to create FixTweet's streamlined API responses */ +const populateUserProperties = async ( + response: GraphQLUserResponse, + language: string | undefined + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const apiUser = {} as APIUser; + + const user = response.data.user.result; + /* Populating a lot of the basics */ + apiUser.url = `${Constants.TWITTER_ROOT}/${user.legacy.screen_name}`; + apiUser.id = user.id; + apiUser.followers = user.legacy.followers_count; + apiUser.following = user.legacy.friends_count; + apiUser.likes = user.legacy.favourites_count; + apiUser.tweets = user.legacy.statuses_count; + apiUser.name = user.legacy.name; + apiUser.screen_name = user.legacy.screen_name; + apiUser.description = user.legacy.description; + apiUser.location = user.legacy.location; + apiUser.verified = user.legacy.verified; + apiUser.avatar_url = user.legacy.profile_image_url_https; + + return apiUser; +}; + +const writeDataPoint = ( + event: FetchEvent, + language: string | undefined, + returnCode: string, + flags?: InputFlags +) => { + console.log('Writing data point...'); + if (typeof AnalyticsEngine !== 'undefined') { + const flagString = + Object.keys(flags || {}) + // @ts-expect-error - TypeScript doesn't like iterating over the keys, but that's OK + .filter(flag => flags?.[flag])[0] || 'standard'; + + AnalyticsEngine.writeDataPoint({ + blobs: [ + event.request.cf?.colo as string /* Datacenter location */, + event.request.cf?.country as string /* Country code */, + event.request.headers.get('user-agent') ?? + '' /* User agent (for aggregating bots calling) */, + returnCode /* Return code */, + flagString /* Type of request */, + language ?? '' /* For translate feature */ + ], + doubles: [0 /* NSFW media = 1, No NSFW Media = 0 */], + indexes: [event.request.headers.get('cf-ray') ?? '' /* CF Ray */] + }); + } +}; + +/* API for Twitter profiles (Users) + Used internally by FixTweet's embed service, or + available for free using api.fxtwitter.com. */ +export const userAPI = async ( + username: string, + language: string | undefined, + event: FetchEvent, + flags?: InputFlags +): Promise => { + const userResponse = await fetchUser(username, event); + + /* Creating the response objects */ + const response: UserAPIResponse = { code: 200, message: 'OK' } as UserAPIResponse; + const apiTweet: APIUser = (await populateUserProperties( + userResponse, + language + )) as APIUser; + + /* Finally, staple the User to the response and return it */ + response.user = apiTweet; + + writeDataPoint(event, language, 'OK', flags); + + return response; +}; diff --git a/src/fetch.ts b/src/fetch.ts index 51c1554..e520e5f 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -203,3 +203,40 @@ export const fetchConversation = async ( } )) as TimelineBlobPartial; }; + +export const fetchUser = async ( + username: string, + event: FetchEvent, + useElongator = false +): Promise => { + return (await twitterFetch( + `${Constants.TWITTER_ROOT}/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?variables=${ + encodeURIComponent( + JSON.stringify({ + screen_name: username, + withSafetyModeUserFields: true + }) + ) + }&features=${encodeURIComponent( + JSON.stringify({ + blue_business_profile_image_shape_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: true + }) + )}`, + event, + useElongator, + // Validator function + (_res: unknown) => { + const response = _res as GraphQLUserResponse; + return !(response?.data?.user?.result?.__typename !== 'User' || typeof response.data.user.result.legacy === 'undefined'); + /* + return !( + typeof conversation.globalObjects === 'undefined' && + (typeof conversation.errors === 'undefined' || + conversation.errors?.[0]?.code === 239) + ); + */ + } + )) as GraphQLUserResponse; +}; diff --git a/src/server.ts b/src/server.ts index 2a287a6..6b8899c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -7,6 +7,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; +import { handleProfile } from './user'; const router = Router(); @@ -140,18 +141,100 @@ const statusRequest = async ( } }; -/* Redirects to user profile when linked. - We don't do any fancy special embeds yet, just Twitter default embeds. */ -const profileRequest = async (request: IRequest) => { - const { handle } = request.params; +/* Handler for User Profiles */ +const profileRequest = async (request: IRequest, event: FetchEvent, + flags: InputFlags = {}) => { + const { handle, language } = request.params; const url = new URL(request.url); + const userAgent = request.headers.get('User-Agent') || ''; + + /* User Agent matching for embed generators, bots, crawlers, and other automated + tools. It's pretty all-encompassing. Note that Firefox/92 is in here because + Discord sometimes uses the following UA: + + Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 + + I'm not sure why that specific one, it's pretty weird, but this edge case ensures + stuff keeps working. + + On the very rare off chance someone happens to be using specifically Firefox 92, + the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ + const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null; /* If not a valid screen name, we redirect to project GitHub */ if (handle.match(/\w{1,15}/gi)?.[0] !== handle) { return Response.redirect(Constants.REDIRECT_URL, 302); - } else { - return Response.redirect(`${Constants.TWITTER_ROOT}${url.pathname}`, 302); } + const username = handle.match(/\w{1,15}/gi)?.[0]; + /* Check if request is to api.fxtwitter.com, or the tweet is appended with .json + Note that unlike TwitFix, FixTweet will never generate embeds for .json, and + in fact we only support .json because it's what people using TwitFix API would + be used to. */ + if ( + url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null || + Constants.API_HOST_LIST.includes(url.hostname) + ) { + console.log('JSON API request'); + flags.api = true; + } + + /* Direct media or API access bypasses bot check, returning same response regardless of UA */ + if (isBotUA || flags.direct || flags.api) { + if (isBotUA) { + console.log(`Matched bot UA ${userAgent}`); + } else { + console.log('Bypass bot check'); + } + + /* This throws the necessary data to handleStatus (in status.ts) */ + const profileResponse = await handleProfile( + handle.match(/\w{1,15}/gi)?.[0] || '', + userAgent, + flags, + language, + event + ); + + /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */ + if (profileResponse.response) { + console.log('handleProfile sent response'); + return profileResponse.response; + } else if (profileResponse.text) { + console.log('handleProfile sent embed'); + /* TODO This check has purpose in the original handleStatus handler, but I'm not sure if this edge case can happen here */ + if (!isBotUA) { + return Response.redirect(`${Constants.TWITTER_ROOT}/${handle}`, 302); + } + + let headers = Constants.RESPONSE_HEADERS; + + if (profileResponse.cacheControl) { + headers = { ...headers, 'cache-control': profileResponse.cacheControl }; + } + + /* Return the response containing embed information */ + return new Response(profileResponse.text, { + headers: headers, + status: 200 + }); + } else { + /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ + return new Response(Strings.ERROR_UNKNOWN, { + headers: Constants.RESPONSE_HEADERS, + status: 500 + }); + } + } else { + /* A human has clicked a fxtwitter.com/:screen_name link! + Obviously we just need to redirect to the user directly.*/ + console.log('Matched human UA', userAgent); + return Response.redirect( + `${Constants.TWITTER_ROOT}/${handle}`, + 302 + ); + } + + }; const genericTwitterRedirect = async (request: IRequest) => { diff --git a/src/strings.ts b/src/strings.ts index 037b8d7..b6e8b4f 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -148,6 +148,7 @@ This is caused by Twitter API downtime or a new bug. Try again in a little while ERROR_API_FAIL: 'Tweet failed to load due to an API error :(', ERROR_PRIVATE: `Due to Twitter API changes, some NSFW Tweets are currently being blocked. We are currently looking into a workaround. ๐Ÿ™`, ERROR_TWEET_NOT_FOUND: `Sorry, that Tweet doesn't exist :(`, + ERROR_USER_NOT_FOUND: `Sorry, that Tweet doesn't exist :(`, ERROR_UNKNOWN: `Unknown error occurred, sorry about that :(`, TWITFIX_API_SUNSET: `The original TwitFix API has been sunset. To learn more about the FixTweet API, check out ; // {}, + "is_blue_verified": boolean; // false, + "profile_image_shape": 'Circle' | 'Square'; // "Circle", + "legacy": { + "created_at": string; // "Sat Sep 26 17:20:55 +0000 2015", + "default_profile": boolean // false, + "default_profile_image": boolean // false, + "description": string; // "dangered wolf#3621 https://t.co/eBTS4kksMw", + "entities": { + "description": { + "urls": { + "display_url": string; // "t.me/dangeredwolf", + "expanded_url": string; // "http://t.me/dangeredwolf", + "url": string; // "https://t.co/eBTS4kksMw", + "indices": [ + 19, + 42 + ] + }[] + } + }, + "fast_followers_count": 0, + "favourites_count": number; // 126708, + "followers_count": number; // 4996, + "friends_count": number; // 2125, + "has_custom_timelines": boolean; // true, + "is_translator": boolean; // false, + "listed_count": number; // 69, + "location": string; // "they/them", + "media_count": number; // 20839, + "name": string; // "dangered wolf", + "normal_followers_count": number; // 4996, + "pinned_tweet_ids_str": string[]; // Array of tweet ids + "possibly_sensitive": boolean; // false, + "profile_banner_url": string; // "https://pbs.twimg.com/profile_banners/3784131322/1658599775", + "profile_image_url_https": string; // "https://pbs.twimg.com/profile_images/1555638673705783299/3gaaetxC_normal.jpg", + "profile_interstitial_type": string; // "", + "screen_name": string; // "dangeredwolf", + "statuses_count": number; // 108222, + "translator_type": string; // "regular", + "verified": boolean; // false, + "withheld_in_countries": [] + }, + "professional": { + "rest_id": string; // "1508134739420536845", + "professional_type": string; // "Creator", + "category": [ + { + "id": number; // 354, + "name": string // "Fish & Chips Restaurant", + "icon_name": string; // "IconBriefcaseStroke" + } + ] + }, + "legacy_extended_profile": { + birthdate?: { + day: number; // 7, + month: number; // 1, + visibility: string; // "Public" + year: number; // 2000 + year_visibility: string; // "Public" + }; + profile_image_shape: string; // "Circle", + rest_id: string; // "3784131322", + }, + "is_profile_translatable": false, + "verification_info": { + reason: { + description: { + entities: { + from_index: number; // 98, + ref: { + url: string; // "https://help.twitter.com/managing-your-account/about-twitter-verified-accounts", + url_type: string; // "ExternalUrl" + }; + to_index: number; // 108 + }[]; + text: string; // "This account is verified because itโ€™s subscribed to Twitter Blue or is a legacy verified account. Learn more" + } + } + }, + "business_account": {} + +} \ No newline at end of file diff --git a/src/types/types.d.ts b/src/types/types.d.ts index 21c3a60..84070f2 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -40,12 +40,18 @@ interface VerticalSize { secondHeight: number; } -interface APIResponse { +interface TweetAPIResponse { code: number; message: string; tweet?: APITweet; } +interface UserAPIResponse { + code: number; + message: string; + user?: APIUser; +} + interface APITranslate { text: string; source_lang: string; @@ -141,3 +147,22 @@ interface APITweet { twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; } + +interface APIUser { + id: string; + name: string; + screen_name: string; + avatar_url: string; + banner_url: string; + avatar_color: string; + description: string; + location: string; + url: string; + protected: boolean; + verified: boolean; + followers: number; + following: number; + tweets: number; + likes: number; + joined: string; +} \ No newline at end of file diff --git a/src/user.ts b/src/user.ts new file mode 100644 index 0000000..ac4037d --- /dev/null +++ b/src/user.ts @@ -0,0 +1,53 @@ +import { Constants } from './constants'; +import { handleQuote } from './helpers/quote'; +import { formatNumber, sanitizeText } from './helpers/utils'; +import { Strings } from './strings'; +import { getAuthorText } from './helpers/author'; +import { userAPI } from './api/user'; + +export const returnError = (error: string): StatusResponse => { + return { + text: Strings.BASE_HTML.format({ + lang: '', + headers: [ + ``, + `` + ].join('') + }) + }; +}; + +/* Handler for Twitter users */ +export const handleProfile = async ( + username: string, + userAgent?: string, + flags?: InputFlags, + language?: string, + event?: FetchEvent +): Promise => { + console.log('Direct?', flags?.direct); + + const api = await userAPI(username, language, event as FetchEvent); + const user = api?.user as APIUser; + + /* Catch this request if it's an API response */ + // For now we just always return the API response while testing + if (flags?.api || true) { + return { + response: new Response(JSON.stringify(api), { + headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }, + status: api.code + }) + }; + } + + /* If there was any errors fetching the User, we'll return it */ + switch (api.code || true) { + case 401: + return returnError(Strings.ERROR_PRIVATE); + case 404: + return returnError(Strings.ERROR_USER_NOT_FOUND); + case 500: + return returnError(Strings.ERROR_API_FAIL); + } +}; diff --git a/test/index.test.ts b/test/index.test.ts index 95d4062..ffbba6b 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -312,3 +312,52 @@ test('API fetch poll Tweet', async () => { expect(choices[3].count).toEqual(31706); expect(choices[3].percentage).toEqual(58); }); + +test('API fetch user', async () => { + const result = await cacheWrapper( + new Request('https://api.fxtwitter.com/wazbat', { + method: 'GET', + headers: botHeaders + }) + ); + expect(result.status).toEqual(200); + const response = (await result.json()) as APIResponse; + expect(response).toBeTruthy(); + expect(response.code).toEqual(200); + expect(response.message).toEqual('OK'); + + const tweet = response.tweet as APITweet; + expect(tweet).toBeTruthy(); + expect(tweet.url).toEqual('https://twitter.com/Twitter/status/1055475950543167488'); + expect(tweet.id).toEqual('1055475950543167488'); + expect(tweet.text).toEqual('A poll:'); + expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter'); + expect(tweet.author.name).toBeTruthy(); + expect(tweet.author.avatar_url).toBeTruthy(); + expect(tweet.author.banner_url).toBeTruthy(); + expect(tweet.author.avatar_color).toBeTruthy(); + expect(tweet.twitter_card).toEqual('tweet'); + expect(tweet.created_at).toEqual('Thu Oct 25 15:07:31 +0000 2018'); + expect(tweet.created_timestamp).toEqual(1540480051); + expect(tweet.lang).toEqual('en'); + expect(tweet.replying_to).toBeNull(); + expect(tweet.poll).toBeTruthy(); + const poll = tweet.poll as APIPoll; + expect(poll.ends_at).toEqual('2018-10-26T03:07:30Z'); + expect(poll.time_left_en).toEqual('Final results'); + expect(poll.total_votes).toEqual(54703); + + const choices = poll.choices as APIPollChoice[]; + expect(choices[0].label).toEqual('Yesssss'); + expect(choices[0].count).toEqual(14773); + expect(choices[0].percentage).toEqual(27); + expect(choices[1].label).toEqual('No'); + expect(choices[1].count).toEqual(3618); + expect(choices[1].percentage).toEqual(6.6); + expect(choices[2].label).toEqual('Maybe?'); + expect(choices[2].count).toEqual(4606); + expect(choices[2].percentage).toEqual(8.4); + expect(choices[3].label).toEqual('Just show me the results'); + expect(choices[3].count).toEqual(31706); + expect(choices[3].percentage).toEqual(58); +});