diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 1972e50..82f22d9 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -10,7 +10,10 @@ config(); const gitCommit = execSync('git rev-parse --short HEAD').toString().trim(); const gitCommitFull = execSync('git rev-parse HEAD').toString().trim(); const gitUrl = execSync('git remote get-url origin').toString().trim(); -const gitBranch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim().replace(/[\\\/]/g, '-'); +const gitBranch = execSync('git rev-parse --abbrev-ref HEAD') + .toString() + .trim() + .replace(/[\\\/]/g, '-'); let workerName = 'fixtweet'; diff --git a/src/api/status.ts b/src/api/status.ts deleted file mode 100644 index 569b8c1..0000000 --- a/src/api/status.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { renderCard } from '../helpers/card'; -import { Constants } from '../constants'; -import { fetchConversation } from '../fetch'; -import { linkFixer } from '../helpers/linkFixer'; -import { handleMosaic } from '../helpers/mosaic'; -import { translateTweet } from '../helpers/translate'; -import { unescapeText } from '../helpers/utils'; -import { processMedia } from '../helpers/media'; -import { convertToApiUser } from './user'; -import { isGraphQLTweet } from '../helpers/graphql'; - -/* This function does the heavy lifting of processing data from Twitter API - and using it to create FixTweet's streamlined API responses */ -const populateTweetProperties = async ( - tweet: GraphQLTweet, - conversation: TweetResultsByRestIdResult, // TimelineBlobPartial, - language: string | undefined - // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { - const apiTweet = {} as APITweet; - - /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. - It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ - - if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { - tweet = tweet.result; - } - - if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { - tweet.core = tweet.tweet.core; - } - - if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { - tweet.legacy = tweet.tweet?.legacy; - } - - if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { - tweet.views = tweet?.tweet?.views; - } - - if (typeof tweet.core === 'undefined') { - console.log('Tweet still not valid', tweet); - return null; - } - - /* With v2 conversation API we re-add the user object ot the tweet because - Twitter stores it separately in the conversation API. This is to consolidate - it in case a user appears multiple times in a thread. */ - const graphQLUser = tweet.core.user_results.result; - const apiUser = convertToApiUser(graphQLUser); - - /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ - const id = tweet.rest_id ?? tweet.legacy.id_str; - - /* Populating a lot of the basics */ - apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; - apiTweet.id = id; - apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); - apiTweet.author = { - id: apiUser.id, - name: apiUser.name, - screen_name: apiUser.screen_name, - avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', - avatar_color: null, - banner_url: apiUser.banner_url || '', - description: apiUser.description || '', - location: apiUser.location || '', - url: apiUser.url || '', - followers: apiUser.followers, - following: apiUser.following, - joined: apiUser.joined, - tweets: apiUser.tweets, - likes: apiUser.likes, - protected: apiUser.protected, - birthday: apiUser.birthday, - website: apiUser.website - }; - apiTweet.replies = tweet.legacy.reply_count; - apiTweet.retweets = tweet.legacy.retweet_count; - apiTweet.likes = tweet.legacy.favorite_count; - apiTweet.color = null; - apiTweet.twitter_card = 'tweet'; - apiTweet.created_at = tweet.legacy.created_at; - apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; - - apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; - - if (tweet.views.state === 'EnabledWithCount') { - apiTweet.views = parseInt(tweet.views.count || '0') ?? null; - } else { - apiTweet.views = null; - } - console.log('note_tweet', JSON.stringify(tweet.note_tweet)); - const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; - - if (noteTweetText) { - tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; - tweet.legacy.entities.hashtags = - tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; - tweet.legacy.entities.symbols = - tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; - - console.log('We meet the conditions to use new note tweets'); - apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); - apiTweet.is_note_tweet = true; - } else { - apiTweet.is_note_tweet = false; - } - - if (tweet.legacy.lang !== 'unk') { - apiTweet.lang = tweet.legacy.lang; - } else { - apiTweet.lang = null; - } - - apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; - apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; - - const mediaList = Array.from( - tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] - ); - - // console.log('tweet', JSON.stringify(tweet)); - - /* Populate this Tweet's media */ - mediaList.forEach(media => { - const mediaObject = processMedia(media); - if (mediaObject) { - apiTweet.media = apiTweet.media || {}; - apiTweet.media.all = apiTweet.media?.all || []; - apiTweet.media.all.push(mediaObject); - - if (mediaObject.type === 'photo') { - apiTweet.twitter_card = 'summary_large_image'; - apiTweet.media.photos = apiTweet.media.photos || []; - apiTweet.media.photos.push(mediaObject); - } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { - apiTweet.twitter_card = 'player'; - apiTweet.media.videos = apiTweet.media.videos || []; - apiTweet.media.videos.push(mediaObject); - } else { - console.log('Unknown media type', mediaObject.type); - } - } - }); - - /* Grab color palette data */ - /* - if (mediaList[0]?.ext_media_color?.palette) { - apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); - } - */ - - /* Handle photos and mosaic if available */ - if ((apiTweet?.media?.photos?.length || 0) > 1) { - const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); - if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { - apiTweet.media.mosaic = mosaic; - } - } - - // Add Tweet source but remove the link HTML tag - if (tweet.source) { - apiTweet.source = (tweet.source || '').replace( - /(.+?)<\/a>/, - '$2' - ); - } - - /* Populate a Twitter card */ - - if (tweet.card) { - const card = renderCard(tweet.card); - if (card.external_media) { - apiTweet.media = apiTweet.media || {}; - apiTweet.media.external = card.external_media; - } - if (card.poll) { - apiTweet.poll = card.poll; - } - } - - /* Workaround: Force player card by default for videos */ - /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' - The mediaList however can set it to something else. TODO: Reimplement as enums */ - // @ts-expect-error see above comment - if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') { - apiTweet.twitter_card = 'player'; - } - - /* If a language is specified in API or by user, let's try translating it! */ - if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { - console.log(`Attempting to translate Tweet to ${language}...`); - const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); - if (translateAPI !== null && translateAPI?.translation) { - apiTweet.translation = { - text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), - source_lang: translateAPI?.sourceLanguage || '', - target_lang: translateAPI?.destinationLanguage || '', - source_lang_en: translateAPI?.localizedSourceLanguage || '' - }; - } - } - - return apiTweet; -}; - -const writeDataPoint = ( - event: FetchEvent, - language: string | undefined, - nsfw: boolean, - returnCode: string, - flags?: InputFlags -) => { - console.log('Writing data point...'); - if (typeof AnalyticsEngine !== 'undefined') { - const flagString = - Object.keys(flags || {}) - // @ts-expect-error - TypeScript doesn't like iterating over the keys, but that's OK - .filter(flag => flags?.[flag])[0] || 'standard'; - - AnalyticsEngine.writeDataPoint({ - blobs: [ - event.request.cf?.colo as string /* Datacenter location */, - event.request.cf?.country as string /* Country code */, - event.request.headers.get('user-agent') ?? - '' /* User agent (for aggregating bots calling) */, - returnCode /* Return code */, - flagString /* Type of request */, - language ?? '' /* For translate feature */ - ], - doubles: [nsfw ? 1 : 0 /* NSFW media = 1, No NSFW Media = 0 */], - indexes: [event.request.headers.get('cf-ray') ?? '' /* CF Ray */] - }); - } -}; - -/* API for Twitter statuses (Tweets) - Used internally by FixTweet's embed service, or - available for free using api.fxtwitter.com. */ -export const statusAPI = async ( - status: string, - language: string | undefined, - event: FetchEvent, - flags?: InputFlags -): Promise => { - const res = await fetchConversation(status, event); - const tweet = res.data?.tweetResult?.result; - if (!tweet) { - return { code: 404, message: 'NOT_FOUND' }; - } - /* We're handling this in the actual fetch code now */ - - // if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { - // wasMediaBlockedNSFW = true; - // res = await fetchConversation(status, event, true); - // } - - // console.log(JSON.stringify(tweet)) - - if (tweet.__typename === 'TweetUnavailable') { - if ((tweet as { reason: string })?.reason === 'Protected') { - writeDataPoint(event, language, false, 'PRIVATE_TWEET', flags); - return { code: 401, message: 'PRIVATE_TWEET' }; - // } else if (tweet.reason === 'NsfwLoggedOut') { - // // API failure as elongator should have handled this - // writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); - // return { code: 500, message: 'API_FAIL' }; - } else { - // Api failure at parsing status - writeDataPoint(event, language, false, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; - } - } - // If the tweet is not a graphQL tweet something went wrong - if (!isGraphQLTweet(tweet)) { - console.log('Tweet was not a valid tweet', tweet); - writeDataPoint(event, language, false, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; - } - - /* - if (tweet.retweeted_status_id_str) { - tweet = conversation?.globalObjects?.tweets?.[tweet.retweeted_status_id_str] || {}; - } - */ - - if (!tweet) { - return { code: 404, message: 'NOT_FOUND' }; - } - /* Creating the response objects */ - const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse; - const apiTweet: APITweet = (await populateTweetProperties(tweet, res, language)) as APITweet; - - /* We found a quote tweet, let's process that too */ - const quoteTweet = tweet.quoted_status_result; - if (quoteTweet) { - apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet; - /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.twitter_card === 'tweet' && apiTweet.quote !== null) { - apiTweet.twitter_card = apiTweet.quote.twitter_card; - } - } - - /* Finally, staple the Tweet to the response and return it */ - response.tweet = apiTweet; - - writeDataPoint(event, language, false, 'OK', flags); - - return response; -}; diff --git a/src/api/user.ts b/src/api/user.ts index 45885eb..cf3aab7 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -10,12 +10,16 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.followers = user.legacy.followers_count; apiUser.following = user.legacy.friends_count; apiUser.likes = user.legacy.favourites_count; + // @ts-expect-error `tweets` is only part of legacy API apiUser.tweets = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description); - apiUser.location = user.legacy.location; - apiUser.banner_url = user.legacy.profile_banner_url; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`; + apiUser.description = user.legacy.description + ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) + : ''; + apiUser.location = user.legacy.location ? user.legacy.location : ''; + apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : ''; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; @@ -85,6 +89,9 @@ export const userAPI = async ( const response: UserAPIResponse = { code: 200, message: 'OK' } as UserAPIResponse; const apiUser: APIUser = (await populateUserProperties(userResponse)) as APIUser; + /* Currently, we haven't rolled this out as it's part of the proto-v2 API */ + delete apiUser.global_screen_name; + /* Finally, staple the User to the response and return it */ response.user = apiUser; diff --git a/src/constants.ts b/src/constants.ts index 5c8ff7e..0b0bfa5 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,7 @@ export const Constants = { RELEASE_NAME: RELEASE_NAME, API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`, TWITTER_ROOT: 'https://twitter.com', + TWITTER_GLOBAL_NAME_ROOT: 'twitter.com', TWITTER_API_ROOT: 'https://api.twitter.com', BOT_UA_REGEX: /bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi, @@ -58,8 +59,8 @@ export const Constants = { RESPONSE_HEADERS: { 'allow': 'OPTIONS, GET, PURGE, HEAD', 'content-type': 'text/html;charset=UTF-8', - 'x-powered-by': '🏳️‍⚧️ Trans Rights', - 'cache-control': 'max-age=3600' // Can be overriden in some cases, like poll tweets + 'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`, + 'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets }, API_RESPONSE_HEADERS: { 'access-control-allow-origin': '*', @@ -68,8 +69,3 @@ export const Constants = { POLL_TWEET_CACHE: 'max-age=60', DEFAULT_COLOR: '#10A3FF' }; - -if (typeof TEST !== 'undefined') { - /* Undici gets angry about unicode headers, this is a workaround. */ - Constants.RESPONSE_HEADERS['x-powered-by'] = 'Trans Rights'; -} diff --git a/src/embed/status.ts b/src/embed/status.ts index d38198f..1a58182 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -3,10 +3,11 @@ import { handleQuote } from '../helpers/quote'; import { formatNumber, sanitizeText, truncateWithEllipsis } from '../helpers/utils'; import { Strings } from '../strings'; import { getAuthorText } from '../helpers/author'; -import { statusAPI } from '../api/status'; import { renderPhoto } from '../render/photo'; import { renderVideo } from '../render/video'; import { renderInstantView } from '../render/instantview'; +import { constructTwitterThread } from '../providers/twitter/conversation'; +import { IRequest } from 'itty-router'; export const returnError = (error: string): StatusResponse => { return { @@ -24,17 +25,55 @@ export const returnError = (error: string): StatusResponse => { Like Twitter, we use the terminologies interchangably. */ export const handleStatus = async ( status: string, - mediaNumber?: number, - userAgent?: string, - flags?: InputFlags, - language?: string, - event?: FetchEvent + mediaNumber: number | undefined, + userAgent: string, + flags: InputFlags, + language: string, + event: FetchEvent, + request: IRequest // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { console.log('Direct?', flags?.direct); - const api = await statusAPI(status, language, event as FetchEvent, flags); - const tweet = api?.tweet as APITweet; + console.log('event', event) + + let fetchWithThreads = false; + + if (request?.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) { + fetchWithThreads = true; + } + + const thread = await constructTwitterThread( + status, + fetchWithThreads, + request, + language, + flags?.api ?? false + ); + + const tweet = thread?.post as APITweet; + + const api = { + code: thread.code, + message: '', + tweet: tweet + }; + + switch (api.code) { + case 200: + api.message = 'OK'; + break; + case 401: + api.message = 'PRIVATE_TWEET'; + break; + case 404: + api.message = 'NOT_FOUND'; + break; + case 500: + console.log(api); + api.message = 'API_FAIL'; + break; + } /* Catch this request if it's an API response */ if (flags?.api) { @@ -46,6 +85,10 @@ export const handleStatus = async ( }; } + if (tweet === null) { + return returnError(Strings.ERROR_TWEET_NOT_FOUND); + } + /* If there was any errors fetching the Tweet, we'll return it */ switch (api.code) { case 401: @@ -121,7 +164,7 @@ export const handleStatus = async ( const headers = [ ``, ``, - ``, + ``, ``, ``, `` @@ -200,8 +243,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card === 'player') { - tweet.twitter_card = 'summary_large_image'; + if (tweet.embed_card === 'player') { + tweet.embed_card = 'summary_large_image'; } break; case 'video': @@ -217,8 +260,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card !== 'player') { - tweet.twitter_card = 'player'; + if (tweet.embed_card !== 'player') { + tweet.embed_card = 'player'; } /* This Tweet has a video to render. */ break; @@ -345,7 +388,7 @@ export const handleStatus = async ( and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */ const text = useIV ? sanitizeText(newText).replace(/\n/g, '
') : sanitizeText(newText); - const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card; + const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card; /* Push basic headers relating to author, Tweet text, and site name */ headers.push( @@ -360,7 +403,7 @@ export const handleStatus = async ( authorText = `↪ Replying to @${tweet.replying_to}`; /* We'll assume it's a thread if it's a reply to themselves */ } else if ( - tweet.replying_to === tweet.author.screen_name && + tweet.replying_to?.screen_name === tweet.author.screen_name && authorText === Strings.DEFAULT_AUTHOR_TEXT ) { authorText = `↪ A part of @${tweet.author.screen_name}'s thread`; diff --git a/src/experiments.ts b/src/experiments.ts index b5c084c..39640bc 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -1,6 +1,7 @@ export enum Experiment { ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT', - ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API' + ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API', + TWEET_DETAIL_API = 'TWEET_DETAIL_API' } type ExperimentConfig = { @@ -13,12 +14,17 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = { [Experiment.ELONGATOR_BY_DEFAULT]: { name: 'Elongator by default', description: 'Enable Elongator by default (guest token lockout bypass)', - percentage: 0.6 + percentage: 1 }, [Experiment.ELONGATOR_PROFILE_API]: { name: 'Elongator profile API', description: 'Use Elongator to load profiles', percentage: 0 + }, + [Experiment.TWEET_DETAIL_API]: { + name: 'Tweet detail API', + description: 'Use Tweet Detail API (where available with elongator)', + percentage: 0.75 } }; diff --git a/src/fetch.ts b/src/fetch.ts index 39c6b6a..07d19d8 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,7 +1,6 @@ import { Constants } from './constants'; import { Experiment, experimentCheck } from './experiments'; import { generateUserAgent } from './helpers/useragent'; -import { isGraphQLTweet } from './helpers/graphql'; const API_ATTEMPTS = 3; let wasElongatorDisabled = false; @@ -26,7 +25,8 @@ export const twitterFetch = async ( Experiment.ELONGATOR_BY_DEFAULT, typeof TwitterProxy !== 'undefined' ), - validateFunction: (response: unknown) => boolean + validateFunction: (response: unknown) => boolean, + elongatorRequired = false ): Promise => { let apiAttempts = 0; let newTokenGenerated = false; @@ -163,6 +163,11 @@ export const twitterFetch = async ( /* We'll usually only hit this if we get an invalid response from Twitter. It's uncommon, but it happens */ console.error('Unknown error while fetching from API', e); + /* Elongator returns strings to communicate downstream errors */ + if (String(e).indexOf('Status not found')) { + console.log('Tweet was not found'); + return {}; + } !useElongator && event && event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); @@ -181,7 +186,6 @@ export const twitterFetch = async ( !wasElongatorDisabled && !useElongator && typeof TwitterProxy !== 'undefined' && - // @ts-expect-error This is safe due to optional chaining (response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason === 'NsfwLoggedOut' ) { @@ -201,6 +205,10 @@ export const twitterFetch = async ( if (!validateFunction(response)) { console.log('Failed to fetch response, got', JSON.stringify(response)); + if (elongatorRequired) { + console.log('Elongator was required, but we failed to fetch a valid response'); + return {}; + } if (useElongator) { console.log('Elongator request failed to validate, trying again without it'); wasElongatorDisabled = true; @@ -232,87 +240,6 @@ export const twitterFetch = async ( return {}; }; -export const fetchConversation = async ( - status: string, - event: FetchEvent, - useElongator = experimentCheck( - Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' - ) -): Promise => { - return (await twitterFetch( - `${ - Constants.TWITTER_ROOT - }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( - JSON.stringify({ - tweetId: status, - withCommunity: false, - includePromotedContent: false, - withVoice: false - }) - )}&features=${encodeURIComponent( - JSON.stringify({ - creator_subscriptions_tweet_preview_api_enabled: true, - tweetypie_unmention_optimization_enabled: true, - responsive_web_edit_tweet_api_enabled: true, - graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, - view_counts_everywhere_api_enabled: true, - longform_notetweets_consumption_enabled: true, - responsive_web_twitter_article_tweet_consumption_enabled: false, - tweet_awards_web_tipping_enabled: false, - freedom_of_speech_not_reach_fetch_enabled: true, - standardized_nudges_misinfo: true, - tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, - longform_notetweets_rich_text_read_enabled: true, - longform_notetweets_inline_media_enabled: true, - responsive_web_graphql_exclude_directive_enabled: true, - verified_phone_label_enabled: false, - responsive_web_media_download_video_enabled: false, - responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, - responsive_web_graphql_timeline_navigation_enabled: true, - responsive_web_enhance_cards_enabled: false - }) - )}&fieldToggles=${encodeURIComponent( - JSON.stringify({ - withArticleRichContentState: true - }) - )}`, - event, - useElongator, - (_conversation: unknown) => { - const conversation = _conversation as TweetResultsByRestIdResult; - // If we get a not found error it's still a valid response - const tweet = conversation.data?.tweetResult?.result; - if (isGraphQLTweet(tweet)) { - return true; - } - console.log('invalid graphql tweet'); - if ( - !tweet && - typeof conversation.data?.tweetResult === 'object' && - Object.keys(conversation.data?.tweetResult || {}).length === 0 - ) { - console.log('tweet was not found'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { - console.log('tweet is nsfw'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { - console.log('tweet is protected'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable') { - console.log('generic tweet unavailable error'); - return true; - } - // Final clause for checking if it's valid is if there's errors - return Array.isArray(conversation.errors); - } - )) as TweetResultsByRestIdResult; -}; - export const fetchUser = async ( username: string, event: FetchEvent, @@ -359,6 +286,7 @@ export const fetchUser = async ( conversation.errors?.[0]?.code === 239) ); */ - } + }, + false )) as GraphQLUserResponse; }; diff --git a/src/helpers/author.ts b/src/helpers/author.ts index bf0a4a8..dde6c44 100644 --- a/src/helpers/author.ts +++ b/src/helpers/author.ts @@ -3,13 +3,13 @@ import { formatNumber } from './utils'; /* The embed "author" text we populate with replies, retweets, and likes unless it's a video */ export const getAuthorText = (tweet: APITweet): string | null => { /* Build out reply, retweet, like counts */ - if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) { + if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) { let authorText = ''; if (tweet.replies > 0) { authorText += `${formatNumber(tweet.replies)} 💬 `; } - if (tweet.retweets > 0) { - authorText += `${formatNumber(tweet.retweets)} 🔁 `; + if (tweet.reposts > 0) { + authorText += `${formatNumber(tweet.reposts)} 🔁 `; } if (tweet.likes > 0) { authorText += `${formatNumber(tweet.likes)} ❤️ `; @@ -28,13 +28,13 @@ export const getAuthorText = (tweet: APITweet): string | null => { /* The embed "author" text we populate with replies, retweets, and likes unless it's a video */ export const getSocialTextIV = (tweet: APITweet): string | null => { /* Build out reply, retweet, like counts */ - if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) { + if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) { let authorText = ''; if (tweet.replies > 0) { authorText += `💬 ${formatNumber(tweet.replies)} `; } - if (tweet.retweets > 0) { - authorText += `🔁 ${formatNumber(tweet.retweets)} `; + if (tweet.reposts > 0) { + authorText += `🔁 ${formatNumber(tweet.reposts)} `; } if (tweet.likes > 0) { authorText += `❤️ ${formatNumber(tweet.likes)} `; diff --git a/src/helpers/mosaic.ts b/src/helpers/mosaic.ts index e686269..0e96315 100644 --- a/src/helpers/mosaic.ts +++ b/src/helpers/mosaic.ts @@ -13,7 +13,7 @@ const getDomain = (twitterId: string): string | null => { hash = (hash << 5) - hash + char; } return mosaicDomains[Math.abs(hash) % mosaicDomains.length]; -} +}; /* Handler for mosaic (multi-image combiner) */ export const handleMosaic = async ( diff --git a/src/helpers/quote.ts b/src/helpers/quote.ts index f72a009..d1b4a0a 100644 --- a/src/helpers/quote.ts +++ b/src/helpers/quote.ts @@ -1,7 +1,7 @@ import { Strings } from '../strings'; /* Helper for Quote Tweets */ -export const handleQuote = (quote: APITweet): string | null => { +export const handleQuote = (quote: APIPost): string | null => { console.log('Quoting status ', quote.id); let str = `\n`; diff --git a/src/helpers/useragent.ts b/src/helpers/useragent.ts index 512d9bb..d09d536 100644 --- a/src/helpers/useragent.ts +++ b/src/helpers/useragent.ts @@ -1,6 +1,6 @@ /* We keep this value up-to-date for making our requests to Twitter as indistinguishable from normal user traffic as possible. */ -const fakeChromeVersion = 116; +const fakeChromeVersion = 118; const platformWindows = 'Windows NT 10.0; Win64; x64'; const platformMac = 'Macintosh; Intel Mac OS X 10_15_7'; const platformLinux = 'X11; Linux x86_64'; diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts new file mode 100644 index 0000000..20c0e3e --- /dev/null +++ b/src/providers/twitter/conversation.ts @@ -0,0 +1,513 @@ +import { IRequest } from 'itty-router'; +import { Constants } from '../../constants'; +import { twitterFetch } from '../../fetch'; +import { buildAPITweet } from './processor'; +import { Experiment, experimentCheck } from '../../experiments'; +import { isGraphQLTweet } from '../../helpers/graphql'; + +export const fetchTweetDetail = async ( + status: string, + event: FetchEvent, + useElongator = typeof TwitterProxy !== 'undefined', + cursor: string | null = null +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( + JSON.stringify({ + focalTweetId: status, + referrer: 'home', + with_rux_injections: false, + includePromotedContent: false, + withCommunity: true, + withBirdwatchNotes: true, + withQuickPromoteEligibilityTweetFields: false, + withVoice: false, + withV2Timeline: true, + cursor: cursor + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_home_pinned_timelines_enabled: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_media_download_video_enabled: true, + responsive_web_enhance_cards_enabled: true + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + (_conversation: unknown) => { + const conversation = _conversation as TweetDetailResult; + const tweet = findTweetInBucket( + status, + processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions) + ); + if (tweet && isGraphQLTweet(tweet)) { + return true; + } + console.log('invalid graphql tweet', conversation); + const firstInstruction = ( + conversation.data?.threaded_conversation_with_injections_v2 + .instructions?.[0] as TimelineAddEntriesInstruction + )?.entries?.[0]; + if ( + ( + (firstInstruction as { content: GraphQLTimelineItem })?.content + ?.itemContent as GraphQLTimelineTweet + )?.tweet_results?.result?.__typename === 'TweetTombstone' + ) { + console.log('tweet is private'); + return true; + } + + return Array.isArray(conversation?.errors); + }, + true + )) as TweetDetailResult; +}; + +export const fetchByRestId = async ( + status: string, + event: FetchEvent, + useElongator = experimentCheck( + Experiment.ELONGATOR_BY_DEFAULT, + typeof TwitterProxy !== 'undefined' + ) +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( + JSON.stringify({ + tweetId: status, + withCommunity: false, + includePromotedContent: false, + withVoice: false + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + creator_subscriptions_tweet_preview_api_enabled: true, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_media_download_video_enabled: false, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_enhance_cards_enabled: false + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + (_conversation: unknown) => { + const conversation = _conversation as TweetResultsByRestIdResult; + // If we get a not found error it's still a valid response + const tweet = conversation.data?.tweetResult?.result; + if (isGraphQLTweet(tweet)) { + return true; + } + console.log('invalid graphql tweet'); + if ( + !tweet && + typeof conversation.data?.tweetResult === 'object' && + Object.keys(conversation.data?.tweetResult || {}).length === 0 + ) { + console.log('tweet was not found'); + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { + console.log('tweet is nsfw'); + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + console.log('tweet is protected'); + return true; + } + if (tweet?.__typename === 'TweetUnavailable') { + console.log('generic tweet unavailable error'); + return true; + } + // Final clause for checking if it's valid is if there's errors + return Array.isArray(conversation.errors); + }, + false + )) as TweetResultsByRestIdResult; +}; + +const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => { + const bucket: GraphQLProcessBucket = { + tweets: [], + cursors: [] + }; + instructions.forEach?.(instruction => { + if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { + // @ts-expect-error Use entries or moduleItems depending on the type + (instruction?.entries ?? instruction.moduleItems).forEach(_entry => { + const entry = _entry as + | GraphQLTimelineTweetEntry + | GraphQLConversationThread + | GraphQLModuleTweetEntry; + const content = + (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + if (content.__typename === 'TimelineTimelineItem') { + const itemContentType = content.itemContent?.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = content.itemContent.tweet_results.result.__typename; + if (entryType === 'Tweet') { + bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); + } + } else if ( + (content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule' + ) { + content.items.forEach(item => { + const itemContentType = item.item.itemContent.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = item.item.itemContent.tweet_results.result.__typename; + if (entryType === 'Tweet') { + bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push( + item.item.itemContent.tweet_results.result.tweet as GraphQLTweet + ); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); + } + }); + } + }); + } + }); + + return bucket; +}; + +const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { + return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; +}; + +const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { + return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); +}; + +const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { + const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); + if (!tweet) { + console.log('uhhh, we could not even find that tweet, dunno how that happened'); + return -1; + } + return bucket.tweets.findIndex( + _tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str + ); +}; + +const consolidateCursors = ( + oldCursors: GraphQLTimelineCursor[], + newCursors: GraphQLTimelineCursor[] +): GraphQLTimelineCursor[] => { + /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ + return oldCursors.map(cursor => { + const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); + if (newCursor) { + return newCursor; + } + return cursor; + }); +}; + +const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { + return tweets.filter( + tweet => + tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id + ); +}; + +/* Fetch and construct a Twitter thread */ +export const constructTwitterThread = async ( + id: string, + processThread = false, + request: IRequest, + language: string | undefined, + legacyAPI = false +): Promise => { + console.log('language', language); + + let response: TweetDetailResult | TweetResultsByRestIdResult | null = null; + let post: APITweet; + /* We can use TweetDetail on elongator accounts to increase per-account rate limit. + We also use TweetDetail to process threads (WIP) + + Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why. + I'll figure out why eventually, but for now just don't use TweetDetail for this. */ + if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { + console.log('Using TweetDetail for request...'); + response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult; + + console.log(response); + + const firstInstruction = ( + response.data?.threaded_conversation_with_injections_v2 + .instructions?.[0] as TimelineAddEntriesInstruction + )?.entries?.[0]; + if ( + ( + (firstInstruction as { content: GraphQLTimelineItem })?.content + ?.itemContent as GraphQLTimelineTweet + )?.tweet_results?.result?.__typename === 'TweetTombstone' /* If a tweet is private */ + ) { + console.log('tweet is private'); + return { post: null, thread: null, author: null, code: 401 }; + } else if (!response.data) { + return { post: null, thread: null, author: null, code: 404 }; + } + } + + /* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */ + if (!response) { + console.log('Using TweetResultsByRestId for request...'); + response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult; + + const result = response?.data?.tweetResult?.result as GraphQLTweet; + + if (typeof result === 'undefined') { + return { post: null, thread: null, author: null, code: 404 }; + } + + const buildPost = await buildAPITweet(result, language, false, legacyAPI); + + if ((buildPost as FetchResults).status === 401) { + return { post: null, thread: null, author: null, code: 401 }; + } else if (buildPost === null) { + return { post: null, thread: null, author: null, code: 404 }; + } + + post = buildPost as APITweet; + + return { post: post, thread: null, author: post.author, code: 200 }; + } + + const bucket = processResponse( + response.data.threaded_conversation_with_injections_v2.instructions + ); + const originalTweet = findTweetInBucket(id, bucket); + + /* Don't bother processing thread on a null tweet */ + if (originalTweet === null) { + return { post: null, thread: null, author: null, code: 404 }; + } + + post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet; + + if (post === null) { + return { post: null, thread: null, author: null, code: 404 }; + } + + const author = post.author; + + /* If we're not processing threads, let's be done here */ + if (!processThread) { + return { post: post, thread: null, author: author, code: 200 }; + } + + const threadTweets = [originalTweet]; + bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet); + + let currentId = id; + + /* Process tweets that are following the current one in the thread */ + while (findNextTweet(currentId, bucket) !== -1) { + const index = findNextTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log( + 'adding next tweet to thread', + newCurrentId, + 'from', + currentId, + 'at index', + index, + 'in bucket' + ); + + threadTweets.push(tweet); + + currentId = newCurrentId; + + console.log('Current index', index, 'of', bucket.tweets.length); + + /* Reached the end of the current list of tweets in thread) */ + if (index >= bucket.tweets.length - 1) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find( + cursor => cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore' + ); + console.log('current cursors: ', bucket.cursors); + if (!cursor) { + console.log('No cursor present, stopping pagination down'); + break; + } + console.log('Cursor present, fetching more tweets down'); + + let loadCursor: TweetDetailResult; + + try { + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); + + if ( + typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === + 'undefined' + ) { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch (e) { + console.log('Error fetching cursor', e); + break; + } + + const cursorResponse = processResponse( + loadCursor.data.threaded_conversation_with_injections_v2.instructions + ); + bucket.tweets = bucket.tweets.concat( + filterBucketTweets(cursorResponse.tweets, originalTweet) + ); + /* Remove old cursor and add new bottom cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of next tweet:', findNextTweet(currentId, bucket)); + } + + currentId = id; + + while (findPreviousTweet(currentId, bucket) !== -1) { + const index = findPreviousTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log( + 'adding previous tweet to thread', + newCurrentId, + 'from', + currentId, + 'at index', + index, + 'in bucket' + ); + + threadTweets.unshift(tweet); + + currentId = newCurrentId; + + if (index === 0) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find( + cursor => cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore' + ); + console.log('current cursors: ', bucket.cursors); + if (!cursor) { + console.log('No cursor present, stopping pagination up'); + break; + } + console.log('Cursor present, fetching more tweets up'); + + let loadCursor: TweetDetailResult; + + try { + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); + + if ( + typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === + 'undefined' + ) { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch (e) { + console.log('Error fetching cursor', e); + break; + } + const cursorResponse = processResponse( + loadCursor.data.threaded_conversation_with_injections_v2.instructions + ); + bucket.tweets = cursorResponse.tweets.concat( + filterBucketTweets(bucket.tweets, originalTweet) + ); + /* Remove old cursor and add new top cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + + // console.log('updated bucket of tweets', bucket.tweets); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket)); + } + + const socialThread: SocialThread = { + post: post, + thread: [], + author: author, + code: 200 + }; + + threadTweets.forEach(async tweet => { + socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet); + }); + + return socialThread; +}; + +export const threadAPIProvider = async (request: IRequest) => { + const { id } = request.params; + + const processedResponse = await constructTwitterThread(id, true, request, undefined); + + return new Response(JSON.stringify(processedResponse), { + headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } + }); +}; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts new file mode 100644 index 0000000..c11b436 --- /dev/null +++ b/src/providers/twitter/processor.ts @@ -0,0 +1,265 @@ +import { renderCard } from '../../helpers/card'; +import { Constants } from '../../constants'; +import { linkFixer } from '../../helpers/linkFixer'; +import { handleMosaic } from '../../helpers/mosaic'; +// import { translateTweet } from '../../helpers/translate'; +import { unescapeText } from '../../helpers/utils'; +import { processMedia } from '../../helpers/media'; +import { convertToApiUser } from '../../api/user'; +import { translateTweet } from '../../helpers/translate'; + +export const buildAPITweet = async ( + tweet: GraphQLTweet, + language: string | undefined, + threadPiece = false, + legacyAPI = false + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const apiTweet = {} as APITweet; + + /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. + It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ + + if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { + tweet = tweet.result; + } + + if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { + tweet.core = tweet.tweet.core; + } + + if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { + tweet.legacy = tweet.tweet?.legacy; + } + + if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { + tweet.views = tweet?.tweet?.views; + } + + if (typeof tweet.core === 'undefined') { + console.log('Tweet still not valid', tweet); + if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + return { status: 401 }; + } else { + return { status: 404 }; + } + } + + const graphQLUser = tweet.core.user_results.result; + const apiUser = convertToApiUser(graphQLUser); + + /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ + const id = tweet.rest_id ?? tweet.legacy.id_str; + + /* Populating a lot of the basics */ + apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; + apiTweet.id = id; + apiTweet.text = unescapeText( + linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '') + ); + if (!threadPiece) { + apiTweet.author = { + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, + avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, + banner_url: apiUser.banner_url, + description: apiUser.description, + location: apiUser.location, + url: apiUser.url, + followers: apiUser.followers, + following: apiUser.following, + joined: apiUser.joined, + posts: apiUser.posts, + likes: apiUser.likes, + protected: apiUser.protected, + birthday: apiUser.birthday, + website: apiUser.website + }; + } + apiTweet.replies = tweet.legacy.reply_count; + if (legacyAPI) { + // @ts-expect-error Use retweets for legacy API + apiTweet.retweets = tweet.legacy.retweet_count; + + // @ts-expect-error `tweets` is only part of legacy API + apiTweet.author.tweets = apiTweet.author.posts; + // @ts-expect-error Part of legacy API that we no longer are able to track + apiTweet.author.avatar_color = null; + // @ts-expect-error Use retweets for legacy API + delete apiTweet.reposts; + // @ts-expect-error Use tweets and not posts for legacy API + delete apiTweet.author.posts; + delete apiTweet.author.global_screen_name; + } else { + apiTweet.reposts = tweet.legacy.retweet_count; + apiTweet.author.global_screen_name = apiUser.global_screen_name; + } + apiTweet.likes = tweet.legacy.favorite_count; + apiTweet.embed_card = 'tweet'; + apiTweet.created_at = tweet.legacy.created_at; + apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; + + apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; + + if (tweet.views.state === 'EnabledWithCount') { + apiTweet.views = parseInt(tweet.views.count || '0') ?? null; + } else { + apiTweet.views = null; + } + console.log('note_tweet', JSON.stringify(tweet.note_tweet)); + const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; + + if (noteTweetText) { + tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; + tweet.legacy.entities.hashtags = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; + tweet.legacy.entities.symbols = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; + + console.log('We meet the conditions to use new note tweets'); + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); + apiTweet.is_note_tweet = true; + } else { + apiTweet.is_note_tweet = false; + } + + if (tweet.legacy.lang !== 'unk') { + apiTweet.lang = tweet.legacy.lang; + } else { + apiTweet.lang = null; + } + + if (legacyAPI) { + // @ts-expect-error Use replying_to string for legacy API + apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + // @ts-expect-error Use replying_to_status string for legacy API + apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; + } else if (tweet.legacy.in_reply_to_screen_name) { + apiTweet.replying_to = { + screen_name: tweet.legacy.in_reply_to_screen_name || null, + post: tweet.legacy.in_reply_to_status_id_str || null + }; + } else { + apiTweet.replying_to = null; + } + + apiTweet.media = {}; + + /* We found a quote tweet, let's process that too */ + const quoteTweet = tweet.quoted_status_result; + if (quoteTweet) { + apiTweet.quote = (await buildAPITweet( + quoteTweet, + language, + threadPiece, + legacyAPI + )) as APITweet; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { + apiTweet.embed_card = apiTweet.quote.embed_card; + } + } + + const mediaList = Array.from( + tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] + ); + + // console.log('tweet', JSON.stringify(tweet)); + + /* Populate this Tweet's media */ + mediaList.forEach(media => { + const mediaObject = processMedia(media); + if (mediaObject) { + apiTweet.media.all = apiTweet.media?.all ?? []; + apiTweet.media?.all?.push(mediaObject); + if (mediaObject.type === 'photo') { + apiTweet.embed_card = 'summary_large_image'; + apiTweet.media.photos = apiTweet.media?.photos ?? []; + apiTweet.media.photos?.push(mediaObject); + } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { + apiTweet.embed_card = 'player'; + apiTweet.media.videos = apiTweet.media?.videos ?? []; + apiTweet.media.videos?.push(mediaObject); + } else { + console.log('Unknown media type', mediaObject.type); + } + } + }); + + /* Grab color palette data */ + /* + if (mediaList[0]?.ext_media_color?.palette) { + apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); + } + */ + + /* Handle photos and mosaic if available */ + if ((apiTweet?.media.photos?.length || 0) > 1 && !threadPiece) { + const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); + if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { + apiTweet.media.mosaic = mosaic; + } + } + + // Add Tweet source but remove the link HTML tag + if (tweet.source) { + apiTweet.source = (tweet.source || '').replace( + /(.+?)<\/a>/, + '$2' + ); + } + + /* Populate a Twitter card */ + + if (tweet.card) { + const card = renderCard(tweet.card); + if (card.external_media) { + apiTweet.media.external = card.external_media; + } + if (card.poll) { + apiTweet.poll = card.poll; + } + } + + if ( + apiTweet.media?.videos && + apiTweet.media?.videos.length > 0 && + apiTweet.embed_card !== 'player' + ) { + apiTweet.embed_card = 'player'; + } + + console.log('language?', language) + + /* If a language is specified in API or by user, let's try translating it! */ + if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { + console.log(`Attempting to translate Tweet to ${language}...`); + const translateAPI = await translateTweet(tweet, '', language); + if (translateAPI !== null && translateAPI?.translation) { + apiTweet.translation = { + text: unescapeText( + linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '') + ), + source_lang: translateAPI?.sourceLanguage || '', + target_lang: translateAPI?.destinationLanguage || '', + source_lang_en: translateAPI?.localizedSourceLanguage || '' + }; + } + } + + if (legacyAPI) { + // @ts-expect-error Use twitter_card for legacy API + apiTweet.twitter_card = apiTweet.embed_card; + // @ts-expect-error Part of legacy API that we no longer are able to track + apiTweet.color = null; + // @ts-expect-error Use twitter_card for legacy API + delete apiTweet.embed_card; + if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) { + // @ts-expect-error media is not required in legacy API if empty + delete apiTweet.media; + } + } + + return apiTweet; +}; diff --git a/src/render/instantview.ts b/src/render/instantview.ts index 2ffcbec..8779695 100644 --- a/src/render/instantview.ts +++ b/src/render/instantview.ts @@ -4,7 +4,7 @@ import { getSocialTextIV } from '../helpers/author'; import { sanitizeText } from '../helpers/utils'; import { Strings } from '../strings'; -const populateUserLinks = (tweet: APITweet, text: string): string => { +const populateUserLinks = (tweet: APIPost, text: string): string => { /* TODO: Maybe we can add username splices to our API so only genuinely valid users are linked? */ text.match(/@(\w{1,15})/g)?.forEach(match => { const username = match.replace('@', ''); @@ -16,7 +16,7 @@ const populateUserLinks = (tweet: APITweet, text: string): string => { return text; }; -const generateTweetMedia = (tweet: APITweet): string => { +const generateTweetMedia = (tweet: APIPost): string => { let media = ''; if (tweet.media?.all?.length) { tweet.media.all.forEach(mediaItem => { @@ -117,7 +117,7 @@ const truncateSocialCount = (count: number): string => { } }; -const generateTweetFooter = (tweet: APITweet, isQuote = false): string => { +const generateTweetFooter = (tweet: APIPost, isQuote = false): string => { const { author } = tweet; let description = author.description; @@ -131,7 +131,7 @@ const generateTweetFooter = (tweet: APITweet, isQuote = false): string => { {aboutSection} `.format({ - socialText: getSocialTextIV(tweet) || '', + socialText: getSocialTextIV(tweet as APITweet) || '', viewOriginal: !isQuote ? `View original post` : notApplicableComment, aboutSection: isQuote ? '' @@ -156,18 +156,18 @@ const generateTweetFooter = (tweet: APITweet, isQuote = false): string => { joined: author.joined ? `📆 ${formatDate(new Date(author.joined))}` : '', following: truncateSocialCount(author.following), followers: truncateSocialCount(author.followers), - tweets: truncateSocialCount(author.tweets) + tweets: truncateSocialCount(author.posts) }) }); }; -const generateTweet = (tweet: APITweet, isQuote = false): string => { +const generateTweet = (tweet: APIPost, isQuote = false): string => { let text = paragraphify(sanitizeText(tweet.text), isQuote); text = htmlifyLinks(text); text = htmlifyHashtags(text); text = populateUserLinks(tweet, text); - const translatedText = getTranslatedText(tweet, isQuote); + const translatedText = getTranslatedText(tweet as APITweet, isQuote); return ` {quoteHeader} diff --git a/src/render/photo.ts b/src/render/photo.ts index 8813a09..4c78824 100644 --- a/src/render/photo.ts +++ b/src/render/photo.ts @@ -35,6 +35,8 @@ export const renderPhoto = ( } } + console.log('photo!', photo); + if (photo.type === 'mosaic_photo' && !isOverrideMedia) { instructions.addHeaders = [ ``, diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 35bce9b..e8e693d 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -354,7 +354,8 @@ type GraphQLTweetLegacy = { type GraphQLTweet = { // Workaround result: GraphQLTweet; - __typename: 'Tweet' | 'TweetUnavailable'; + __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; + reason: string; // used for errors rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -444,37 +445,79 @@ type TweetTombstone = { }; }; }; + +type GraphQLTimelineTweet = { + item: 'TimelineTweet'; + __typename: 'TimelineTweet'; + tweet_results: { + result: GraphQLTweet | TweetTombstone; + }; +}; + +type GraphQLTimelineCursor = { + cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; + itemType: 'TimelineTimelineCursor'; + value: string; + __typename: 'TimelineTimelineCursor'; +}; + +interface GraphQLBaseTimeline { + entryType: string; + __typename: string; +} + +type GraphQLTimelineItem = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineItem'; + __typename: 'TimelineTimelineItem'; + itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; +}; + +type GraphQLTimelineModule = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineModule'; + __typename: 'TimelineTimelineModule'; + items: { + entryId: `conversationthread-${number}-tweet-${number}`; + item: GraphQLTimelineItem; + }[]; +}; + type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ entryId: `tweet-${number}`; // "tweet-1674824189176590336" sortIndex: string; - content: { - entryType: 'TimelineTimelineItem'; - __typename: 'TimelineTimelineItem'; - itemContent: { - item: 'TimelineTweet'; - __typename: 'TimelineTweet'; - tweet_results: { - result: GraphQLTweet | TweetTombstone; - }; - }; - }; + content: GraphQLTimelineItem; }; + +type GraphQLModuleTweetEntry = { + /** The entryID contains the tweet ID */ + sortIndex: string; + item: GraphQLTimelineItem | GraphQLTimelineModule; +}; + type GraphQLConversationThread = { entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336" sortIndex: string; + content: GraphQLTimelineModule; }; type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction; +type ThreadInstruction = + | TimelineAddEntriesInstruction + | TimelineTerminateTimelineInstruction + | TimelineAddModulesInstruction; -type TimeLineAddEntriesInstruction = { +type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; entries: GraphQLTimelineEntry[]; }; -type TimeLineTerminateTimelineInstruction = { +type TimelineAddModulesInstruction = { + type: 'TimelineAddToModule'; + moduleItems: GraphQLTimelineEntry[]; +}; + +type TimelineTerminateTimelineInstruction = { type: 'TimelineTerminateTimeline'; direction: 'Top'; }; @@ -504,10 +547,11 @@ type GraphQLTweetNotFoundResponse = { ]; data: Record; }; -type GraphQLTweetFoundResponse = { +type TweetDetailResult = { + errors?: unknown[]; data: { threaded_conversation_with_injections_v2: { - instructions: V2ThreadInstruction[]; + instructions: ThreadInstruction[]; }; }; }; @@ -516,12 +560,17 @@ type TweetResultsByRestIdResult = { errors?: unknown[]; data?: { tweetResult?: { - result?: - | { - __typename: 'TweetUnavailable'; - reason: 'NsfwLoggedOut' | 'Protected'; - } - | GraphQLTweet; + result?: TweetStub | GraphQLTweet; }; }; }; + +type TweetStub = { + __typename: 'TweetUnavailable'; + reason: 'NsfwLoggedOut' | 'Protected'; +}; + +interface GraphQLProcessBucket { + tweets: GraphQLTweet[]; + cursors: GraphQLTimelineCursor[]; +} diff --git a/src/types/types.d.ts b/src/types/types.d.ts index d6f9f7f..b3465e6 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -43,31 +43,18 @@ interface Request { }; } -interface Size { - width: number; - height: number; -} - -interface HorizontalSize { - width: number; - height: number; - firstWidth: number; - secondWidth: number; -} - -interface VerticalSize { - width: number; - height: number; - firstHeight: number; - secondHeight: number; -} - interface TweetAPIResponse { code: number; message: string; tweet?: APITweet; } +interface SocialPostAPIResponse { + code: number; + message: string; + post?: APITweet; +} + interface UserAPIResponse { code: number; message: string; @@ -81,14 +68,6 @@ interface APITranslate { target_lang: string; } -interface BaseUser { - id?: string; - name?: string; - screen_name?: string; - avatar_url?: string; - banner_url?: string; -} - interface APIExternalMedia { type: 'video'; url: string; @@ -136,7 +115,7 @@ interface APIMosaicPhoto extends APIMedia { }; } -interface APITweet { +interface APIPost { id: string; url: string; text: string; @@ -144,18 +123,14 @@ interface APITweet { created_timestamp: number; likes: number; - retweets: number; + reposts: number; replies: number; - views?: number | null; - color: string | null; - - quote?: APITweet; + quote?: APIPost; poll?: APIPoll; - translation?: APITranslate; author: APIUser; - media?: { + media: { external?: APIExternalMedia; photos?: APIPhoto[]; videos?: APIVideo[]; @@ -166,27 +141,39 @@ interface APITweet { lang: string | null; possibly_sensitive: boolean; - replying_to: string | null; - replying_to_status: string | null; + replying_to: { + screen_name: string | null; + post: string | null; + } | null; - source: string; + source: string | null; - is_note_tweet: boolean; - - twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; + embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; } -interface APIUser extends BaseUser { +interface APITweet extends APIPost { + views?: number | null; + translation?: APITranslate; + + is_note_tweet: boolean; +} + +interface APIUser { + id: string; + name: string; + screen_name: string; + global_screen_name?: string; + avatar_url: string; + banner_url: string; // verified: 'legacy' | 'blue'| 'business' | 'government'; // verified_label: string; description: string; location: string; url: string; - avatar_color?: string | null; protected: boolean; followers: number; following: number; - tweets: number; + posts: number; likes: number; joined: string; website: { @@ -199,3 +186,19 @@ interface APIUser extends BaseUser { year?: number; }; } + +interface SocialPost { + post: APIPost | APITweet | null; + author: APIUser | null; +} + +interface SocialThread { + post: APIPost | APITweet | null; + thread: (APIPost | APITweet)[] | null; + author: APIUser | null; + code: number; +} + +interface FetchResults { + status: number; +} diff --git a/src/worker.ts b/src/worker.ts index 0629235..c907606 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,6 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; +// import { threadAPIProvider } from './providers/twitter/conversation'; declare const globalThis: { fetchCompletedTime: number; @@ -131,6 +132,8 @@ const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputF console.log('Bypass bot check'); } + console.log('event', event) + /* This throws the necessary data to handleStatus (in status.ts) */ const statusResponse = await handleStatus( id?.match(/\d{2,20}/)?.[0] || '0', @@ -138,7 +141,8 @@ const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputF userAgent, flags, language, - event + event, + request ); /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */ @@ -453,6 +457,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); +// router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses) diff --git a/test/index.test.ts b/test/index.test.ts index e34f471..45185e4 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -120,8 +120,10 @@ test('API fetch basic Tweet', async () => { expect(tweet.author.avatar_url).toBeTruthy(); expect(tweet.author.banner_url).toBeTruthy(); expect(tweet.replies).toBeGreaterThan(0); + // @ts-expect-error retweets only in legacy API expect(tweet.retweets).toBeGreaterThan(0); expect(tweet.likes).toBeGreaterThan(0); + // @ts-expect-error twitter_card only in legacy API expect(tweet.twitter_card).toEqual('tweet'); expect(tweet.created_at).toEqual('Tue Mar 21 20:50:14 +0000 2006'); expect(tweet.created_timestamp).toEqual(1142974214); diff --git a/wrangler.example.toml b/wrangler.example.toml index 3df432e..6def706 100644 --- a/wrangler.example.toml +++ b/wrangler.example.toml @@ -3,15 +3,16 @@ account_id = "[CLOUDFLARE_ACCOUNT_ID]" main = "./dist/worker.js" compatibility_date = "2023-08-15" send_metrics = false -services = [ - { binding = "TwitterProxy", service = "elongator" } -] + +# Remove this if not using Cloudflare Analytics Engine analytics_engine_datasets = [ { binding = "AnalyticsEngine" } ] -[build] -command = "npm run build" +# Remove this if not using elongator account proxying +services = [ + { binding = "TwitterProxy", service = "elongator" } +] -[miniflare.globals] -TEST = "true" # Will have unicode character errors in headers if not set to true and running unit tests \ No newline at end of file +[build] +command = "npm run build" \ No newline at end of file