From fe61670e9fbc7e2ab49cbe30af8a89cab399c4aa Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 04:52:14 -0400 Subject: [PATCH] Prettier and additional fixes --- esbuild.config.mjs | 5 +- src/api/user.ts | 6 +- src/embed/status.ts | 18 +- src/experiments.ts | 6 +- src/fetch.ts | 98 ++------- src/helpers/mosaic.ts | 2 +- src/providers/twitter/conversation.ts | 276 +++++++++++++++++--------- src/providers/twitter/processor.ts | 37 +++- src/types/twitterTypes.d.ts | 21 +- src/types/types.d.ts | 8 +- src/worker.ts | 4 +- 11 files changed, 269 insertions(+), 212 deletions(-) diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 1972e50..82f22d9 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -10,7 +10,10 @@ config(); const gitCommit = execSync('git rev-parse --short HEAD').toString().trim(); const gitCommitFull = execSync('git rev-parse HEAD').toString().trim(); const gitUrl = execSync('git remote get-url origin').toString().trim(); -const gitBranch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim().replace(/[\\\/]/g, '-'); +const gitBranch = execSync('git rev-parse --abbrev-ref HEAD') + .toString() + .trim() + .replace(/[\\\/]/g, '-'); let workerName = 'fixtweet'; diff --git a/src/api/user.ts b/src/api/user.ts index c9ba200..dadc83c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -14,8 +14,10 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.tweets = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` - apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : ''; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`; + apiUser.description = user.legacy.description + ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) + : ''; apiUser.location = user.legacy.location ? user.legacy.location : ''; apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : ''; /* diff --git a/src/embed/status.ts b/src/embed/status.ts index ffc930c..8922b20 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -41,7 +41,13 @@ export const handleStatus = async ( fetchWithThreads = true; } - const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined, flags?.api ?? false); + const thread = await constructTwitterThread( + status, + fetchWithThreads, + request, + undefined, + flags?.api ?? false + ); const tweet = thread?.post as APITweet; @@ -51,19 +57,19 @@ export const handleStatus = async ( tweet: tweet }; - switch(api.code) { + switch (api.code) { case 200: - api.message = "OK"; + api.message = 'OK'; break; case 401: - api.message = "PRIVATE_TWEET"; + api.message = 'PRIVATE_TWEET'; break; case 404: - api.message = "NOT_FOUND"; + api.message = 'NOT_FOUND'; break; case 500: console.log(api); - api.message = "API_FAIL"; + api.message = 'API_FAIL'; break; } diff --git a/src/experiments.ts b/src/experiments.ts index f00d53a..0021d51 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -1,7 +1,7 @@ export enum Experiment { ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT', ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API', - TWEET_DETAIL_API = 'TWEET_DETAIL_API', + TWEET_DETAIL_API = 'TWEET_DETAIL_API' } type ExperimentConfig = { @@ -24,8 +24,8 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = { [Experiment.TWEET_DETAIL_API]: { name: 'Tweet detail API', description: 'Use Tweet Detail API (where available with elongator)', - percentage: 0.75 - }, + percentage: 0.5 + } }; export const experimentCheck = (experiment: Experiment, condition = true) => { diff --git a/src/fetch.ts b/src/fetch.ts index 39c6b6a..07d19d8 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,7 +1,6 @@ import { Constants } from './constants'; import { Experiment, experimentCheck } from './experiments'; import { generateUserAgent } from './helpers/useragent'; -import { isGraphQLTweet } from './helpers/graphql'; const API_ATTEMPTS = 3; let wasElongatorDisabled = false; @@ -26,7 +25,8 @@ export const twitterFetch = async ( Experiment.ELONGATOR_BY_DEFAULT, typeof TwitterProxy !== 'undefined' ), - validateFunction: (response: unknown) => boolean + validateFunction: (response: unknown) => boolean, + elongatorRequired = false ): Promise => { let apiAttempts = 0; let newTokenGenerated = false; @@ -163,6 +163,11 @@ export const twitterFetch = async ( /* We'll usually only hit this if we get an invalid response from Twitter. It's uncommon, but it happens */ console.error('Unknown error while fetching from API', e); + /* Elongator returns strings to communicate downstream errors */ + if (String(e).indexOf('Status not found')) { + console.log('Tweet was not found'); + return {}; + } !useElongator && event && event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); @@ -181,7 +186,6 @@ export const twitterFetch = async ( !wasElongatorDisabled && !useElongator && typeof TwitterProxy !== 'undefined' && - // @ts-expect-error This is safe due to optional chaining (response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason === 'NsfwLoggedOut' ) { @@ -201,6 +205,10 @@ export const twitterFetch = async ( if (!validateFunction(response)) { console.log('Failed to fetch response, got', JSON.stringify(response)); + if (elongatorRequired) { + console.log('Elongator was required, but we failed to fetch a valid response'); + return {}; + } if (useElongator) { console.log('Elongator request failed to validate, trying again without it'); wasElongatorDisabled = true; @@ -232,87 +240,6 @@ export const twitterFetch = async ( return {}; }; -export const fetchConversation = async ( - status: string, - event: FetchEvent, - useElongator = experimentCheck( - Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' - ) -): Promise => { - return (await twitterFetch( - `${ - Constants.TWITTER_ROOT - }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( - JSON.stringify({ - tweetId: status, - withCommunity: false, - includePromotedContent: false, - withVoice: false - }) - )}&features=${encodeURIComponent( - JSON.stringify({ - creator_subscriptions_tweet_preview_api_enabled: true, - tweetypie_unmention_optimization_enabled: true, - responsive_web_edit_tweet_api_enabled: true, - graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, - view_counts_everywhere_api_enabled: true, - longform_notetweets_consumption_enabled: true, - responsive_web_twitter_article_tweet_consumption_enabled: false, - tweet_awards_web_tipping_enabled: false, - freedom_of_speech_not_reach_fetch_enabled: true, - standardized_nudges_misinfo: true, - tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, - longform_notetweets_rich_text_read_enabled: true, - longform_notetweets_inline_media_enabled: true, - responsive_web_graphql_exclude_directive_enabled: true, - verified_phone_label_enabled: false, - responsive_web_media_download_video_enabled: false, - responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, - responsive_web_graphql_timeline_navigation_enabled: true, - responsive_web_enhance_cards_enabled: false - }) - )}&fieldToggles=${encodeURIComponent( - JSON.stringify({ - withArticleRichContentState: true - }) - )}`, - event, - useElongator, - (_conversation: unknown) => { - const conversation = _conversation as TweetResultsByRestIdResult; - // If we get a not found error it's still a valid response - const tweet = conversation.data?.tweetResult?.result; - if (isGraphQLTweet(tweet)) { - return true; - } - console.log('invalid graphql tweet'); - if ( - !tweet && - typeof conversation.data?.tweetResult === 'object' && - Object.keys(conversation.data?.tweetResult || {}).length === 0 - ) { - console.log('tweet was not found'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { - console.log('tweet is nsfw'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { - console.log('tweet is protected'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable') { - console.log('generic tweet unavailable error'); - return true; - } - // Final clause for checking if it's valid is if there's errors - return Array.isArray(conversation.errors); - } - )) as TweetResultsByRestIdResult; -}; - export const fetchUser = async ( username: string, event: FetchEvent, @@ -359,6 +286,7 @@ export const fetchUser = async ( conversation.errors?.[0]?.code === 239) ); */ - } + }, + false )) as GraphQLUserResponse; }; diff --git a/src/helpers/mosaic.ts b/src/helpers/mosaic.ts index e686269..0e96315 100644 --- a/src/helpers/mosaic.ts +++ b/src/helpers/mosaic.ts @@ -13,7 +13,7 @@ const getDomain = (twitterId: string): string | null => { hash = (hash << 5) - hash + char; } return mosaicDomains[Math.abs(hash) % mosaicDomains.length]; -} +}; /* Handler for mosaic (multi-image combiner) */ export const handleMosaic = async ( diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index bbcbe37..3fa8956 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -1,23 +1,23 @@ -import { IRequest } from "itty-router"; -import { Constants } from "../../constants"; -import { twitterFetch } from "../../fetch"; -import { buildAPITweet } from "./processor"; -import { Experiment, experimentCheck } from "../../experiments"; -import { isGraphQLTweet } from "../../helpers/graphql"; +import { IRequest } from 'itty-router'; +import { Constants } from '../../constants'; +import { twitterFetch } from '../../fetch'; +import { buildAPITweet } from './processor'; +import { Experiment, experimentCheck } from '../../experiments'; +import { isGraphQLTweet } from '../../helpers/graphql'; export const fetchTweetDetail = async ( status: string, event: FetchEvent, useElongator = typeof TwitterProxy !== 'undefined', cursor: string | null = null -): Promise => { +): Promise => { return (await twitterFetch( `${ Constants.TWITTER_ROOT }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( JSON.stringify({ focalTweetId: status, - referrer: "home", + referrer: 'home', with_rux_injections: false, includePromotedContent: false, withCommunity: true, @@ -58,16 +58,33 @@ export const fetchTweetDetail = async ( event, useElongator, (_conversation: unknown) => { - const conversation = _conversation as GraphQLTweetFoundResponse; - const tweet = findTweetInBucket(status, processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions)); + const conversation = _conversation as TweetDetailResult; + const tweet = findTweetInBucket( + status, + processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions) + ); if (tweet && isGraphQLTweet(tweet)) { return true; } - console.log('invalid graphql tweet'); + console.log('invalid graphql tweet', conversation); + const firstInstruction = ( + conversation.data?.threaded_conversation_with_injections_v2 + .instructions?.[0] as TimelineAddEntriesInstruction + )?.entries?.[0]; + if ( + ( + (firstInstruction as { content: GraphQLTimelineItem })?.content + ?.itemContent as GraphQLTimelineTweet + )?.tweet_results?.result?.__typename === 'TweetTombstone' + ) { + console.log('tweet is private'); + return true; + } return Array.isArray(conversation?.errors); - } - )) as GraphQLTweetFoundResponse; + }, + true + )) as TweetDetailResult; }; export const fetchByRestId = async ( @@ -147,11 +164,11 @@ export const fetchByRestId = async ( } // Final clause for checking if it's valid is if there's errors return Array.isArray(conversation.errors); - } + }, + false )) as TweetResultsByRestIdResult; }; - const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => { const bucket: GraphQLProcessBucket = { tweets: [], @@ -160,13 +177,17 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke instructions.forEach?.(instruction => { if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { // @ts-expect-error Use entries or moduleItems depending on the type - (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => { - const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry - const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + (instruction?.entries ?? instruction.moduleItems).forEach(_entry => { + const entry = _entry as + | GraphQLTimelineTweetEntry + | GraphQLConversationThread + | GraphQLModuleTweetEntry; + const content = + (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; if (content.__typename === 'TimelineTimelineItem') { const itemContentType = content.itemContent?.__typename; if (itemContentType === 'TimelineTweet') { - const entryType = content.itemContent.tweet_results.result.__typename + const entryType = content.itemContent.tweet_results.result.__typename; if (entryType === 'Tweet') { bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); } @@ -176,16 +197,20 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke } else if (itemContentType === 'TimelineTimelineCursor') { bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); } - } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') { - content.items.forEach((item) => { + } else if ( + (content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule' + ) { + content.items.forEach(item => { const itemContentType = item.item.itemContent.__typename; if (itemContentType === 'TimelineTweet') { - const entryType = item.item.itemContent.tweet_results.result.__typename + const entryType = item.item.itemContent.tweet_results.result.__typename; if (entryType === 'Tweet') { bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); } if (entryType === 'TweetWithVisibilityResults') { - bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet); + bucket.tweets.push( + item.item.itemContent.tweet_results.result.tweet as GraphQLTweet + ); } } else if (itemContentType === 'TimelineTimelineCursor') { bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); @@ -194,18 +219,18 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke } }); } - }) + }); return bucket; -} +}; const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; -} +}; const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); -} +}; const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); @@ -213,10 +238,15 @@ const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => console.log('uhhh, we could not even find that tweet, dunno how that happened'); return -1; } - return bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); -} + return bucket.tweets.findIndex( + _tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str + ); +}; -const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { +const consolidateCursors = ( + oldCursors: GraphQLTimelineCursor[], + newCursors: GraphQLTimelineCursor[] +): GraphQLTimelineCursor[] => { /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ return oldCursors.map(cursor => { const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); @@ -225,58 +255,87 @@ const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: Gra } return cursor; }); -} +}; const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { - return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) -} + return tweets.filter( + tweet => + tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id + ); +}; -export const constructTwitterThread = async (id: string, +/* Fetch and construct a Twitter thread */ +export const constructTwitterThread = async ( + id: string, processThread = false, request: IRequest, language: string | undefined, - legacyAPI = false): Promise => { + legacyAPI = false +): Promise => { + console.log('legacyAPI', legacyAPI); - console.log('legacyAPI', legacyAPI) - - let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult; + let response: TweetDetailResult | TweetResultsByRestIdResult | null = null; let post: APITweet; - - if (typeof TwitterProxy === "undefined" || !experimentCheck(Experiment.TWEET_DETAIL_API)) { - console.log('Using TweetResultsByRestId for request...'); - response = await fetchByRestId(id, request.event) as TweetResultsByRestIdResult; - - const result = response?.data?.tweetResult?.result as GraphQLTweet; - - if (typeof result === "undefined") { - return { post: null, thread: null, author: null, code: 404 }; - } - - post = await buildAPITweet(result, language, false, legacyAPI) as APITweet; - - if (post === null) { - return { post: null, thread: null, author: null, code: 404 }; - } - - return { post: post, thread: null, author: post.author, code: 200 }; - } else { + /* We can use TweetDetail on elongator accounts to increase per-account rate limit. + We also use TweetDetail to process threads (WIP) */ + if (typeof TwitterProxy !== 'undefined' && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { console.log('Using TweetDetail for request...'); - response = await fetchTweetDetail(id, request.event) as GraphQLTweetFoundResponse; + response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult; - if (!response.data) { + console.log(response); + + const firstInstruction = ( + response.data?.threaded_conversation_with_injections_v2 + .instructions?.[0] as TimelineAddEntriesInstruction + )?.entries?.[0]; + if ( + ( + (firstInstruction as { content: GraphQLTimelineItem })?.content + ?.itemContent as GraphQLTimelineTweet + )?.tweet_results?.result?.__typename === 'TweetTombstone' /* If a tweet is private */ + ) { + console.log('tweet is private'); + return { post: null, thread: null, author: null, code: 401 }; + } else if (!response.data) { return { post: null, thread: null, author: null, code: 404 }; } } - const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); + /* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */ + if (!response) { + console.log('Using TweetResultsByRestId for request...'); + response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult; + + const result = response?.data?.tweetResult?.result as GraphQLTweet; + + if (typeof result === 'undefined') { + return { post: null, thread: null, author: null, code: 404 }; + } + + const buildPost = await buildAPITweet(result, language, false, legacyAPI); + + if ((buildPost as FetchResults).status === 401) { + return { post: null, thread: null, author: null, code: 401 }; + } else if (buildPost === null) { + return { post: null, thread: null, author: null, code: 404 }; + } + + post = buildPost as APITweet; + + return { post: post, thread: null, author: post.author, code: 200 }; + } + + const bucket = processResponse( + response.data.threaded_conversation_with_injections_v2.instructions + ); const originalTweet = findTweetInBucket(id, bucket); /* Don't bother processing thread on a null tweet */ if (originalTweet === null) { return { post: null, thread: null, author: null, code: 404 }; } - - post = await buildAPITweet(originalTweet, undefined, false, legacyAPI) as APITweet; + + post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet; if (post === null) { return { post: null, thread: null, author: null, code: 404 }; @@ -301,41 +360,58 @@ export const constructTwitterThread = async (id: string, const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; - console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + console.log( + 'adding next tweet to thread', + newCurrentId, + 'from', + currentId, + 'at index', + index, + 'in bucket' + ); threadTweets.push(tweet); currentId = newCurrentId; - console.log('Current index', index, 'of', bucket.tweets.length) + console.log('Current index', index, 'of', bucket.tweets.length); /* Reached the end of the current list of tweets in thread) */ - if (index >= (bucket.tweets.length - 1)) { + if (index >= bucket.tweets.length - 1) { /* See if we have a cursor to fetch more tweets */ - const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore')); - console.log('current cursors: ', bucket.cursors) + const cursor = bucket.cursors.find( + cursor => cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore' + ); + console.log('current cursors: ', bucket.cursors); if (!cursor) { - console.log('No cursor present, stopping pagination down') + console.log('No cursor present, stopping pagination down'); break; } console.log('Cursor present, fetching more tweets down'); - let loadCursor: GraphQLTweetFoundResponse; + let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value) + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); - if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + if ( + typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === + 'undefined' + ) { console.log('Unknown data while fetching cursor', loadCursor); break; } - } catch(e) { + } catch (e) { console.log('Error fetching cursor', e); break; } - const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); - bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet)); + const cursorResponse = processResponse( + loadCursor.data.threaded_conversation_with_injections_v2.instructions + ); + bucket.tweets = bucket.tweets.concat( + filterBucketTweets(cursorResponse.tweets, originalTweet) + ); /* Remove old cursor and add new bottom cursor if necessary */ consolidateCursors(bucket.cursors, cursorResponse.cursors); console.log('updated bucket of cursors', bucket.cursors); @@ -351,36 +427,54 @@ export const constructTwitterThread = async (id: string, const tweet = bucket.tweets[index]; const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; - console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + console.log( + 'adding previous tweet to thread', + newCurrentId, + 'from', + currentId, + 'at index', + index, + 'in bucket' + ); threadTweets.unshift(tweet); currentId = newCurrentId; - + if (index === 0) { /* See if we have a cursor to fetch more tweets */ - const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore')); - console.log('current cursors: ', bucket.cursors) + const cursor = bucket.cursors.find( + cursor => cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore' + ); + console.log('current cursors: ', bucket.cursors); if (!cursor) { - console.log('No cursor present, stopping pagination up') + console.log('No cursor present, stopping pagination up'); break; } console.log('Cursor present, fetching more tweets up'); - let loadCursor: GraphQLTweetFoundResponse; + + let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value) + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); - if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + if ( + typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === + 'undefined' + ) { console.log('Unknown data while fetching cursor', loadCursor); break; } - } catch(e) { + } catch (e) { console.log('Error fetching cursor', e); break; } - const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); - bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet)); + const cursorResponse = processResponse( + loadCursor.data.threaded_conversation_with_injections_v2.instructions + ); + bucket.tweets = cursorResponse.tweets.concat( + filterBucketTweets(bucket.tweets, originalTweet) + ); /* Remove old cursor and add new top cursor if necessary */ consolidateCursors(bucket.cursors, cursorResponse.cursors); @@ -396,14 +490,14 @@ export const constructTwitterThread = async (id: string, thread: [], author: author, code: 200 - } + }; - threadTweets.forEach(async (tweet) => { - socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false) as APITweet); + threadTweets.forEach(async tweet => { + socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet); }); return socialThread; -} +}; export const threadAPIProvider = async (request: IRequest) => { const { id } = request.params; @@ -412,5 +506,5 @@ export const threadAPIProvider = async (request: IRequest) => { return new Response(JSON.stringify(processedResponse), { headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } - }) -} \ No newline at end of file + }); +}; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index e7b39be..ff46146 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -14,7 +14,7 @@ export const buildAPITweet = async ( threadPiece = false, legacyAPI = false // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { const apiTweet = {} as APITweet; /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. @@ -38,7 +38,11 @@ export const buildAPITweet = async ( if (typeof tweet.core === 'undefined') { console.log('Tweet still not valid', tweet); - return null; + if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + return { status: 401 }; + } else { + return { status: 404 }; + } } const graphQLUser = tweet.core.user_results.result; @@ -50,7 +54,9 @@ export const buildAPITweet = async ( /* Populating a lot of the basics */ apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; apiTweet.id = id; - apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); + apiTweet.text = unescapeText( + linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '') + ); if (!threadPiece) { apiTweet.author = { id: apiUser.id, @@ -75,7 +81,7 @@ export const buildAPITweet = async ( if (legacyAPI) { // @ts-expect-error Use retweets for legacy API apiTweet.retweets = tweet.legacy.retweet_count; - + // @ts-expect-error `tweets` is only part of legacy API apiTweet.author.tweets = apiTweet.author.posts; // @ts-expect-error Part of legacy API that we no longer are able to track @@ -139,11 +145,16 @@ export const buildAPITweet = async ( } apiTweet.media = {}; - + /* We found a quote tweet, let's process that too */ const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { - apiTweet.quote = (await buildAPITweet(quoteTweet, language, threadPiece, legacyAPI)) as APITweet; + apiTweet.quote = (await buildAPITweet( + quoteTweet, + language, + threadPiece, + legacyAPI + )) as APITweet; /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { apiTweet.embed_card = apiTweet.quote.embed_card; @@ -211,7 +222,11 @@ export const buildAPITweet = async ( } } - if (apiTweet.media?.videos && apiTweet.media?.videos.length > 0 && apiTweet.embed_card !== 'player') { + if ( + apiTweet.media?.videos && + apiTweet.media?.videos.length > 0 && + apiTweet.embed_card !== 'player' + ) { apiTweet.embed_card = 'player'; } @@ -221,7 +236,9 @@ export const buildAPITweet = async ( const translateAPI = await translateTweet(tweet, '', language); if (translateAPI !== null && translateAPI?.translation) { apiTweet.translation = { - text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + text: unescapeText( + linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '') + ), source_lang: translateAPI?.sourceLanguage || '', target_lang: translateAPI?.destinationLanguage || '', source_lang_en: translateAPI?.localizedSourceLanguage || '' @@ -233,7 +250,7 @@ export const buildAPITweet = async ( // @ts-expect-error Use twitter_card for legacy API apiTweet.twitter_card = apiTweet.embed_card; // @ts-expect-error Part of legacy API that we no longer are able to track - apiTweet.color = null + apiTweet.color = null; // @ts-expect-error Use twitter_card for legacy API delete apiTweet.embed_card; if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) { @@ -243,4 +260,4 @@ export const buildAPITweet = async ( } return apiTweet; -}; \ No newline at end of file +}; diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 5df0992..e8e693d 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -355,6 +355,7 @@ type GraphQLTweet = { // Workaround result: GraphQLTweet; __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; + reason: string; // used for errors rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -451,14 +452,14 @@ type GraphQLTimelineTweet = { tweet_results: { result: GraphQLTweet | TweetTombstone; }; -} +}; type GraphQLTimelineCursor = { cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; itemType: 'TimelineTimelineCursor'; value: string; __typename: 'TimelineTimelineCursor'; -} +}; interface GraphQLBaseTimeline { entryType: string; @@ -469,16 +470,16 @@ type GraphQLTimelineItem = GraphQLBaseTimeline & { entryType: 'TimelineTimelineItem'; __typename: 'TimelineTimelineItem'; itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; -} +}; type GraphQLTimelineModule = GraphQLBaseTimeline & { entryType: 'TimelineTimelineModule'; __typename: 'TimelineTimelineModule'; items: { entryId: `conversationthread-${number}-tweet-${number}`; - item: GraphQLTimelineItem + item: GraphQLTimelineItem; }[]; -} +}; type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ @@ -501,7 +502,10 @@ type GraphQLConversationThread = { type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; +type ThreadInstruction = + | TimelineAddEntriesInstruction + | TimelineTerminateTimelineInstruction + | TimelineAddModulesInstruction; type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; @@ -543,7 +547,7 @@ type GraphQLTweetNotFoundResponse = { ]; data: Record; }; -type GraphQLTweetFoundResponse = { +type TweetDetailResult = { errors?: unknown[]; data: { threaded_conversation_with_injections_v2: { @@ -564,8 +568,7 @@ type TweetResultsByRestIdResult = { type TweetStub = { __typename: 'TweetUnavailable'; reason: 'NsfwLoggedOut' | 'Protected'; -} - +}; interface GraphQLProcessBucket { tweets: GraphQLTweet[]; diff --git a/src/types/types.d.ts b/src/types/types.d.ts index c0fde4b..b3465e6 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -144,7 +144,7 @@ interface APIPost { replying_to: { screen_name: string | null; post: string | null; - } | null + } | null; source: string | null; @@ -197,4 +197,8 @@ interface SocialThread { thread: (APIPost | APITweet)[] | null; author: APIUser | null; code: number; -} \ No newline at end of file +} + +interface FetchResults { + status: number; +} diff --git a/src/worker.ts b/src/worker.ts index 2a6e227..8acd6c1 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,7 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; -import { threadAPIProvider } from './providers/twitter/conversation'; +// import { threadAPIProvider } from './providers/twitter/conversation'; declare const globalThis: { fetchCompletedTime: number; @@ -454,7 +454,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); -router.get('/v2/twitter/thread/:id', threadAPIProvider) +// router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses)