From c196207fc35b57370dcf636ef8196a6d94167828 Mon Sep 17 00:00:00 2001 From: Wazbat Date: Thu, 22 Jun 2023 23:49:44 +0200 Subject: [PATCH 1/8] =?UTF-8?q?=F0=9F=90=9B=20Added=20logic=20for=20non-ex?= =?UTF-8?q?istant=20users?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes an internal server error when users do not exist --- src/api/user.ts | 7 ++++++- src/fetch.ts | 4 ++++ test/index.test.ts | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/api/user.ts b/src/api/user.ts index 593c047..9230fd5 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -60,7 +60,12 @@ export const userAPI = async ( flags?: InputFlags ): Promise => { const userResponse = await fetchUser(username, event); - + if (!userResponse || !Object.keys(userResponse).length) { + return { + code: 404, + message: 'User not found' + }; + } /* Creating the response objects */ const response: UserAPIResponse = { code: 200, message: 'OK' } as UserAPIResponse; const apiUser: APIUser = (await populateUserProperties(userResponse)) as APIUser; diff --git a/src/fetch.ts b/src/fetch.ts index c78c579..6c6ef68 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -231,6 +231,10 @@ export const fetchUser = async ( // Validator function (_res: unknown) => { const response = _res as GraphQLUserResponse; + // If _res.data is an empty object, we have no user + if (!Object.keys(response?.data).length) { + return false; + } return !( response?.data?.user?.result?.__typename !== 'User' || typeof response.data.user.result.legacy === 'undefined' diff --git a/test/index.test.ts b/test/index.test.ts index f5b06f0..6a8af41 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -270,3 +270,18 @@ test('API fetch user', async () => { expect(user.birthday.month).toEqual(3); expect(user.birthday.year).toBeUndefined(); }); + +test.only('API fetch user that does not exist', async () => { + const result = await cacheWrapper( + new Request('https://api.fxtwitter.com/usesaahah123', { + method: 'GET', + headers: botHeaders + }) + ); + expect(result.status).toEqual(404); + const response = (await result.json()) as UserAPIResponse; + expect(response).toBeTruthy(); + expect(response.code).toEqual(404); + expect(response.message).toEqual('User not found'); + expect(response.user).toBeUndefined(); +}); \ No newline at end of file From 8326e7fd8f94a99b3670235c17990e2e144dbdcd Mon Sep 17 00:00:00 2001 From: Wazbat Date: Fri, 30 Jun 2023 19:58:29 +0200 Subject: [PATCH 2/8] Fixed test.only --- test/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/index.test.ts b/test/index.test.ts index 6a8af41..cccd662 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -271,7 +271,7 @@ test('API fetch user', async () => { expect(user.birthday.year).toBeUndefined(); }); -test.only('API fetch user that does not exist', async () => { +test('API fetch user that does not exist', async () => { const result = await cacheWrapper( new Request('https://api.fxtwitter.com/usesaahah123', { method: 'GET', From 703d7744a8e81145b3f87c6349bdde2024479081 Mon Sep 17 00:00:00 2001 From: Wazbat Date: Sat, 1 Jul 2023 00:34:51 +0200 Subject: [PATCH 3/8] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactored=20tweet=20f?= =?UTF-8?q?etching=20to=20use=20graphql=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/status.ts | 170 ++++++++++++++++++------------------ src/api/user.ts | 19 ++-- src/fetch.ts | 77 +++++++++++++--- src/helpers/card.ts | 46 ++++++---- src/helpers/linkFixer.ts | 6 +- src/helpers/translate.ts | 4 +- src/types/twitterTypes.d.ts | 153 ++++++++++++++++++++++++++++++++ src/utils/graphql.ts | 7 ++ 8 files changed, 358 insertions(+), 124 deletions(-) create mode 100644 src/utils/graphql.ts diff --git a/src/api/status.ts b/src/api/status.ts index 545f8fb..05a3e5a 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -7,12 +7,14 @@ import { colorFromPalette } from '../helpers/palette'; import { translateTweet } from '../helpers/translate'; import { unescapeText } from '../helpers/utils'; import { processMedia } from '../helpers/media'; +import { convertToApiUser } from './user'; +import { isGraphQLTweet, isGraphQLTweetNotFoundResponse } from '../utils/graphql'; /* This function does the heavy lifting of processing data from Twitter API and using it to create FixTweet's streamlined API responses */ const populateTweetProperties = async ( - tweet: TweetPartial, - conversation: TimelineBlobPartial, + tweet: GraphQLTweet, + conversation: any, // TimelineBlobPartial, language: string | undefined // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { @@ -21,54 +23,51 @@ const populateTweetProperties = async ( /* With v2 conversation API we re-add the user object ot the tweet because Twitter stores it separately in the conversation API. This is to consolidate it in case a user appears multiple times in a thread. */ - tweet.user = conversation?.globalObjects?.users?.[tweet.user_id_str] || {}; - - const user = tweet.user as UserPartial; - const screenName = user?.screen_name || ''; - const name = user?.name || ''; + const graphQLUser = tweet.core.user_results.result; + const apiUser = convertToApiUser(graphQLUser); /* Populating a lot of the basics */ - apiTweet.url = `${Constants.TWITTER_ROOT}/${screenName}/status/${tweet.id_str}`; - apiTweet.id = tweet.id_str; - apiTweet.text = unescapeText(linkFixer(tweet, tweet.full_text || '')); + apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${tweet.rest_id}`; + apiTweet.id = tweet.rest_id; + apiTweet.text = unescapeText(linkFixer(tweet, tweet.legacy.full_text || '')); apiTweet.author = { - id: tweet.user_id_str, - name: name, - screen_name: screenName, + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, avatar_url: - (user?.profile_image_url_https || '').replace('_normal', '_200x200') || '', - avatar_color: colorFromPalette( + (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', + avatar_color: '0000FF' /* colorFromPalette( tweet.user?.profile_image_extensions_media_color?.palette || [] - ), - banner_url: user?.profile_banner_url || '' + ),*/, + banner_url: apiUser.banner_url || '' }; - apiTweet.replies = tweet.reply_count; - apiTweet.retweets = tweet.retweet_count; - apiTweet.likes = tweet.favorite_count; + apiTweet.replies = tweet.legacy.reply_count; + apiTweet.retweets = tweet.legacy.retweet_count; + apiTweet.likes = tweet.legacy.favorite_count; apiTweet.color = apiTweet.author.avatar_color; apiTweet.twitter_card = 'tweet'; - apiTweet.created_at = tweet.created_at; - apiTweet.created_timestamp = new Date(tweet.created_at).getTime() / 1000; + apiTweet.created_at = tweet.legacy.created_at; + apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; - apiTweet.possibly_sensitive = tweet.possibly_sensitive; + apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; - if (tweet.ext_views?.state === 'EnabledWithCount') { - apiTweet.views = parseInt(tweet.ext_views.count || '0') ?? null; + if (tweet.views.state === 'EnabledWithCount') { + apiTweet.views = parseInt(tweet.views.count || '0') ?? null; } else { apiTweet.views = null; } - if (tweet.lang !== 'unk') { - apiTweet.lang = tweet.lang; + if (tweet.legacy.lang !== 'unk') { + apiTweet.lang = tweet.legacy.lang; } else { apiTweet.lang = null; } - apiTweet.replying_to = tweet.in_reply_to_screen_name || null; - apiTweet.replying_to_status = tweet.in_reply_to_status_id_str || null; - + apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; + const mediaList = Array.from( - tweet.extended_entities?.media || tweet.entities?.media || [] + tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] ); // console.log('tweet', JSON.stringify(tweet)); @@ -94,13 +93,15 @@ const populateTweetProperties = async ( }); /* Grab color palette data */ + /* if (mediaList[0]?.ext_media_color?.palette) { apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); } + */ /* Handle photos and mosaic if available */ if ((apiTweet.media?.photos?.length || 0) > 1) { - const mosaic = await handleMosaic(apiTweet.media?.photos || [], tweet.id_str); + const mosaic = await handleMosaic(apiTweet.media?.photos || [], tweet.rest_id); if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { apiTweet.media.mosaic = mosaic; } @@ -115,6 +116,7 @@ const populateTweetProperties = async ( } /* Populate a Twitter card */ + if (tweet.card) { const card = await renderCard(tweet.card); if (card.external_media) { @@ -128,7 +130,7 @@ const populateTweetProperties = async ( } /* If a language is specified in API or by user, let's try translating it! */ - if (typeof language === 'string' && language.length === 2 && language !== tweet.lang) { + if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { const translateAPI = await translateTweet( tweet, conversation.guestToken || '', @@ -186,59 +188,62 @@ export const statusAPI = async ( event: FetchEvent, flags?: InputFlags ): Promise => { - let conversation = await fetchConversation(status, event); - let tweet = conversation?.globalObjects?.tweets?.[status] || {}; - let wasMediaBlockedNSFW = false; - + let conversation = await fetchConversation(status, event); + let tweet: GraphQLTweet | TweetTombstone; + if (isGraphQLTweetNotFoundResponse(conversation)) { + writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); + return { code: 404, message: 'NOT_FOUND' }; + } + /* Fallback for if Tweet did not load (i.e. NSFW) */ + if (Object.keys(conversation).length === 0) { + // Try again using elongator API proxy + console.log('No Tweet was found, loading again from elongator'); + conversation = await fetchConversation(status, event, true); + if (Object.keys(conversation).length === 0) { + writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); + return { code: 404, message: 'NOT_FOUND' }; + } + // If the tweet now loads, it was probably NSFW + wasMediaBlockedNSFW = true; + } + // Find this specific tweet in the conversation + try { + const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions; + if (!Array.isArray(instructions)) { + console.log(JSON.stringify(conversation, null, 2)); + throw new Error('Invalid instructions'); + } + const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries'); + if (!timelineAddEntries) throw new Error('No valid timeline entries'); + const graphQLTimelineTweetEntry = timelineAddEntries.entries + .find((e): e is GraphQLTimelineTweetEntry => + // TODO Fix this idk what's up with the typings + !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`)); + if (!graphQLTimelineTweetEntry) throw new Error('No tweet entry with'); + tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result; + if (!tweet) throw new Error('No tweet in timeline entry'); + } catch (e) { + // Api failure at parsing status + console.log('Tweet could not be accessed, got conversation ', conversation); + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; + } + // If the tweet is not a graphQL tweet it's a tombstone, return the error to the user + if (!isGraphQLTweet(tweet)) { + console.log('Tweet was not a valid tweet', tweet); + writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags); + return { code: 401, message: 'PRIVATE_TWEET' }; + } + + /* if (tweet.retweeted_status_id_str) { tweet = conversation?.globalObjects?.tweets?.[tweet.retweeted_status_id_str] || {}; } + */ - /* Fallback for if Tweet did not load (i.e. NSFW) */ - if (typeof tweet.full_text === 'undefined') { - if (conversation.timeline?.instructions?.length > 0) { - /* Try again using elongator API proxy */ - console.log('No Tweet was found, loading again from elongator'); - conversation = await fetchConversation(status, event, true); - tweet = conversation?.globalObjects?.tweets?.[status] || {}; - - if (typeof tweet.full_text !== 'undefined') { - console.log('Successfully loaded Tweet using elongator'); - wasMediaBlockedNSFW = true; - } else if ( - typeof tweet.full_text === 'undefined' && - conversation.timeline?.instructions?.length > 0 - ) { - console.log( - 'Tweet could not be accessed with elongator, must be private/suspended, got tweet ', - tweet, - ' conversation ', - conversation - ); - - writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags); - return { code: 401, message: 'PRIVATE_TWEET' }; - } - } else { - /* {"errors":[{"code":34,"message":"Sorry, that page does not exist."}]} */ - if (conversation.errors?.[0]?.code === 34) { - writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); - return { code: 404, message: 'NOT_FOUND' }; - } - - /* Commented this the part below out for now since it seems like atm this check doesn't actually do anything */ - - /* Tweets object is completely missing, smells like API failure */ - // if (typeof conversation?.globalObjects?.tweets === 'undefined') { - // writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); - // return { code: 500, message: 'API_FAIL' }; - // } - - /* If we have no idea what happened then just return API error */ - writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; - } + if (!tweet) { + return { code: 404, message: 'NOT_FOUND' }; } /* Creating the response objects */ @@ -250,8 +255,7 @@ export const statusAPI = async ( )) as APITweet; /* We found a quote tweet, let's process that too */ - const quoteTweet = - conversation.globalObjects?.tweets?.[tweet.quoted_status_id_str || '0'] || null; + const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { apiTweet.quote = (await populateTweetProperties( quoteTweet, diff --git a/src/api/user.ts b/src/api/user.ts index 9230fd5..603d35c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -1,15 +1,8 @@ import { Constants } from '../constants'; import { fetchUser } from '../fetch'; -/* This function does the heavy lifting of processing data from Twitter API - and using it to create FixTweet's streamlined API responses */ -const populateUserProperties = async ( - response: GraphQLUserResponse - // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +export const convertToApiUser = (user: GraphQLUser): APIUser => { const apiUser = {} as APIUser; - - const user = response.data.user.result; /* Populating a lot of the basics */ apiUser.url = `${Constants.TWITTER_ROOT}/${user.legacy.screen_name}`; apiUser.id = user.rest_id; @@ -51,6 +44,16 @@ const populateUserProperties = async ( return apiUser; }; +/* This function does the heavy lifting of processing data from Twitter API + and using it to create FixTweet's streamlined API responses */ +const populateUserProperties = async ( + response: GraphQLUserResponse + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const user = response.data.user.result; + return convertToApiUser(user); +}; + /* API for Twitter profiles (Users) Used internally by FixTweet's embed service, or available for free using api.fxtwitter.com. */ diff --git a/src/fetch.ts b/src/fetch.ts index 6c6ef68..d66d943 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,5 +1,6 @@ import { Constants } from './constants'; import { generateUserAgent } from './helpers/useragent'; +import { isGraphQLTweetNotFoundResponse } from './utils/graphql'; const API_ATTEMPTS = 16; @@ -130,7 +131,12 @@ export const twitterFetch = async ( headers: headers }); } - + /* + If the tweet is nsfw, the body is empty and status is 404 + const raw = await apiRequest?.clone().text(); + console.log('Raw response:', raw); + console.log('Response code:', apiRequest?.status); + */ response = await apiRequest?.json(); } catch (e: unknown) { /* We'll usually only hit this if we get an invalid response from Twitter. @@ -188,20 +194,71 @@ export const fetchConversation = async ( status: string, event: FetchEvent, useElongator = false -): Promise => { +): Promise => { return (await twitterFetch( - `${Constants.TWITTER_API_ROOT}/2/timeline/conversation/${status}.json?${Constants.GUEST_FETCH_PARAMETERS}`, + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/TuC3CinYecrqAyqccUyFhw/TweetDetail?variables=${encodeURIComponent( + JSON.stringify({ + focalTweetId: status, + referrer: 'messages', + with_rux_injections:false, + includePromotedContent:true, + withCommunity:true, + withQuickPromoteEligibilityTweetFields:true, + withArticleRichContent:false, + withBirdwatchNotes:true, + withVoice:true, + withV2Timeline:true + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + rweb_lists_timeline_redesign_enabled:true, + responsive_web_graphql_exclude_directive_enabled:true, + verified_phone_label_enabled:false, + creator_subscriptions_tweet_preview_api_enabled:true, + responsive_web_graphql_timeline_navigation_enabled:true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled:false, + tweetypie_unmention_optimization_enabled:true, + responsive_web_edit_tweet_api_enabled:true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled:true, + view_counts_everywhere_api_enabled:true, + longform_notetweets_consumption_enabled:true, + responsive_web_twitter_article_tweet_consumption_enabled:false, + tweet_awards_web_tipping_enabled:false, + freedom_of_speech_not_reach_fetch_enabled:true, + standardized_nudges_misinfo:true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:true, + longform_notetweets_rich_text_read_enabled:true, + longform_notetweets_inline_media_enabled:true, + responsive_web_media_download_video_enabled:false, + responsive_web_enhance_cards_enabled:false + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + // TODO Figure out what this property does + withArticleRichContentState: false + }) + )}`, event, useElongator, (_conversation: unknown) => { - const conversation = _conversation as TimelineBlobPartial; - return !( - typeof conversation.globalObjects === 'undefined' && - (typeof conversation.errors === 'undefined' || - conversation.errors?.[0]?.code === 239) - ); + const conversation = _conversation as GraphQLTweetDetailResponse; + // If we get a not found error it's still a valid response + if (isGraphQLTweetNotFoundResponse(conversation)) return true; + const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions; + if (!Array.isArray(instructions)) return false; + const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries'); + if (!timelineAddEntries) return false; + const graphQLTimelineTweetEntry = timelineAddEntries.entries + .find((e): e is GraphQLTimelineTweetEntry => + // TODO Fix this idk what's up with the typings + !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`)); + if (!graphQLTimelineTweetEntry) return false; + const tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result; + return !!tweet; } - )) as TimelineBlobPartial; + )) as GraphQLTweetDetailResponse; }; export const fetchUser = async ( diff --git a/src/helpers/card.ts b/src/helpers/card.ts index 865e350..1be6ebe 100644 --- a/src/helpers/card.ts +++ b/src/helpers/card.ts @@ -2,33 +2,43 @@ import { calculateTimeLeftString } from './pollTime'; /* Renders card for polls and non-Twitter video embeds (i.e. YouTube) */ export const renderCard = async ( - card: TweetCard + card: GraphQLTweet['card'] ): Promise<{ poll?: APIPoll; external_media?: APIExternalMedia }> => { - const values = card.binding_values; + // We convert the binding_values array into an object with the legacy format + // TODO Clean this up + const binding_values: Record = {}; + if (Array.isArray(card.legacy.binding_values)) { + card.legacy.binding_values.forEach(value => { + if (value.key && value.value) { + binding_values[value.key] = value.value; + } + }); + } + console.log('rendering card'); - if (typeof values !== 'undefined') { - if (typeof values.choice1_count !== 'undefined') { + if (typeof binding_values !== 'undefined') { + if (typeof binding_values.choice1_count !== 'undefined') { const poll = {} as APIPoll; - poll.ends_at = values.end_datetime_utc?.string_value || ''; + poll.ends_at = binding_values.end_datetime_utc?.string_value || ''; poll.time_left_en = calculateTimeLeftString( - new Date(values.end_datetime_utc?.string_value || '') + new Date(binding_values.end_datetime_utc?.string_value || '') ); const choices: { [label: string]: number } = { - [values.choice1_label?.string_value || '']: parseInt( - values.choice1_count?.string_value || '0' + [binding_values.choice1_label?.string_value || '']: parseInt( + binding_values.choice1_count?.string_value || '0' ), - [values.choice2_label?.string_value || '']: parseInt( - values.choice2_count?.string_value || '0' + [binding_values.choice2_label?.string_value || '']: parseInt( + binding_values.choice2_count?.string_value || '0' ), - [values.choice3_label?.string_value || '']: parseInt( - values.choice3_count?.string_value || '0' + [binding_values.choice3_label?.string_value || '']: parseInt( + binding_values.choice3_count?.string_value || '0' ), - [values.choice4_label?.string_value || '']: parseInt( - values.choice4_count?.string_value || '0' + [binding_values.choice4_label?.string_value || '']: parseInt( + binding_values.choice4_count?.string_value || '0' ) }; @@ -46,17 +56,17 @@ export const renderCard = async ( }); return { poll: poll }; - } else if (typeof values.player_url !== 'undefined') { + } else if (typeof binding_values.player_url !== 'undefined' && binding_values.player_url.string_value) { /* Oh good, a non-Twitter video URL! This enables YouTube embeds and stuff to just work */ return { external_media: { type: 'video', - url: values.player_url.string_value, + url: binding_values.player_url.string_value, width: parseInt( - (values.player_width?.string_value || '1280').replace('px', '') + (binding_values.player_width?.string_value || '1280').replace('px', '') ), // TODO: Replacing px might not be necessary, it's just there as a precaution height: parseInt( - (values.player_height?.string_value || '720').replace('px', '') + (binding_values.player_height?.string_value || '720').replace('px', '') ) } }; diff --git a/src/helpers/linkFixer.ts b/src/helpers/linkFixer.ts index d28bcd9..29a82d3 100644 --- a/src/helpers/linkFixer.ts +++ b/src/helpers/linkFixer.ts @@ -1,7 +1,7 @@ /* Helps replace t.co links with their originals */ -export const linkFixer = (tweet: TweetPartial, text: string): string => { - if (typeof tweet.entities?.urls !== 'undefined') { - tweet.entities?.urls.forEach((url: TcoExpansion) => { +export const linkFixer = (tweet: GraphQLTweet, text: string): string => { + if (Array.isArray(tweet.legacy.entities?.urls) && tweet.legacy.entities.urls.length) { + tweet.legacy.entities.urls.forEach((url: TcoExpansion) => { let newURL = url.expanded_url; if (newURL.match(/^https:\/\/twitter\.com\/i\/web\/status\/\w+/g) !== null) { diff --git a/src/helpers/translate.ts b/src/helpers/translate.ts index fa8921c..79bc2b3 100644 --- a/src/helpers/translate.ts +++ b/src/helpers/translate.ts @@ -2,7 +2,7 @@ import { Constants } from '../constants'; /* Handles translating Tweets when asked! */ export const translateTweet = async ( - tweet: TweetPartial, + tweet: GraphQLTweet, guestToken: string, language: string ): Promise => { @@ -29,7 +29,7 @@ export const translateTweet = async ( try { apiRequest = await fetch( - `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.id_str},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`, + `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`, { method: 'GET', headers: headers diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 2086021..bf85f94 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -304,3 +304,156 @@ type GraphQLUser = { }; }; }; + +type GraphQLTweet = { + __typename: 'Tweet'; + rest_id: string; // "1674824189176590336", + has_birdwatch_notes: false, + core: { + user_results: { + result: GraphQLUser; + } + } + edit_control: unknown, + edit_perspective: unknown, + is_translatable: false, + views: { + count: string; // "562" + state: string; // "EnabledWithCount" + } + source: string; // "Twitter Web App" + quoted_status_result?: GraphQLTweet; + legacy: { + created_at: string; // "Tue Sep 14 20:00:00 +0000 2021" + conversation_id_str: string; // "1674824189176590336" + bookmark_count: number; // 0 + bookmarked: boolean; // false + favorite_count: number; // 28 + full_text: string; // "This is a test tweet" + in_reply_to_screen_name: string; // "username" + in_reply_to_status_id_str: string; // "1674824189176590336" + in_reply_to_user_id_str: string; // "783214" + is_quote_status: boolean; // false + quote_count: number; // 39 + quoted_status_id_str: string; // "1674824189176590336" + quoted_status_permalink: { + url: string; // "https://t.co/aBcDeFgHiJ" + expanded: string; // "https://twitter.com/username/status/1674824189176590336" + display: string; // "twitter.com/username/statu…" + }; + reply_count: number; // 1 + retweet_count: number; // 4 + lang: string; // "en" + possibly_sensitive: boolean; // false + possibly_sensitive_editable: boolean; // false + entities: { + media: { + display_url: string; // "pic.twitter.com/1X2X3X4X5X" + expanded_url: string; // "https://twitter.com/username/status/1674824189176590336/photo/1" "https://twitter.com/username/status/1674824189176590336/video/1" + id_str: string; // "1674824189176590336" + indices: [number, number]; // [number, number] + media_url_https: string; // "https://pbs.twimg.com/media/FAKESCREENSHOT.jpg" With videos appears to be the thumbnail + type: string; // "photo" Seems to be photo even with videos + }[] + user_mentions: unknown[]; + urls: TcoExpansion[]; + hashtags: unknown[]; + symbols: unknown[]; + } + extended_entities: { + media: TweetMedia[] + } + } + card: { + rest_id: string; // "card://1674824189176590336", + legacy: { + binding_values: { + key: `choice${1|2|3|4}_label`|'counts_are_final'|`choice${1|2|3|4}_count`|'last_updated_datetime_utc'|'duration_minutes'|'api'|'card_url' + value: { + string_value: string; // "Option text" + type: 'STRING' + }|{ + boolean_value: boolean; // true + type: 'BOOLEAN' + } + }[] + } + } +} +type TweetTombstone = { + __typename: 'TweetTombstone'; + tombstone: { + __typename: 'TextTombstone'; + text: { + rtl: boolean; // false; + text: string; // "You’re unable to view this Tweet because this account owner limits who can view their Tweets. Learn more" + entities: unknown[]; + } + } +} +type GraphQLTimelineTweetEntry = { + /** The entryID contains the tweet ID */ + entryId: `tweet-${number}`; // "tweet-1674824189176590336" + sortIndex: string; + content: { + entryType: 'TimelineTimelineItem', + __typename: 'TimelineTimelineItem', + itemContent: { + item: 'TimelineTweet', + __typename: 'TimelineTweet', + tweet_results: { + result: GraphQLTweet|TweetTombstone; + } + } + } +} +type GraphQLConversationThread = { + entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336" + sortIndex: string; +} + +type GraphQLTimelineEntry = GraphQLTimelineTweetEntry|GraphQLConversationThread|unknown; + +type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction; + +type TimeLineAddEntriesInstruction = { + type: 'TimelineAddEntries'; + entries: GraphQLTimelineEntry[]; +} + +type TimeLineTerminateTimelineInstruction = { + type: 'TimelineTerminateTimeline'; + direction: 'Top'; +} +type GraphQLTweetNotFoundResponse = { + errors: [{ + message: string; // "_Missing: No status found with that ID" + locations: unknown[]; + path: string[]; // ["threaded_conversation_with_injections_v2"] + extensions: { + name: string; // "GenericError" + source: string; // "Server" + code: number; // 144 + kind: string; // "NonFatal" + tracing: { + trace_id: string; // "2e39ff747de237db" + } + } + code: number; // 144 + kind: string; // "NonFatal" + name: string; // "GenericError" + source: string; // "Server" + tracing: { + trace_id: string; // "2e39ff747de237db" + } + }] + data: Record; +} +type GraphQLTweetFoundResponse = { + data: { + threaded_conversation_with_injections_v2: { + instructions: V2ThreadInstruction[] + } + } +} +type GraphQLTweetDetailResponse = GraphQLTweetFoundResponse | GraphQLTweetNotFoundResponse; \ No newline at end of file diff --git a/src/utils/graphql.ts b/src/utils/graphql.ts new file mode 100644 index 0000000..a6b23cd --- /dev/null +++ b/src/utils/graphql.ts @@ -0,0 +1,7 @@ +export const isGraphQLTweetNotFoundResponse = (response: unknown): response is GraphQLTweetNotFoundResponse => { + return typeof response === 'object' && response !== null && 'errors' in response && Array.isArray(response.errors) && response.errors.length > 0 && 'message' in response.errors[0] && response.errors[0].message === '_Missing: No status found with that ID'; +}; + +export const isGraphQLTweet = (response: unknown): response is GraphQLTweet => { + return typeof response === 'object' && response !== null && '__typename' in response && response.__typename === 'Tweet'; +} \ No newline at end of file From fd0641861b7b0e4c314c9d95e8cc5d5e3ce2284c Mon Sep 17 00:00:00 2001 From: Wazbat Date: Sat, 1 Jul 2023 03:13:16 +0200 Subject: [PATCH 4/8] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Changed=20referrer=20t?= =?UTF-8?q?o=20home?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed unnecessary async --- src/api/status.ts | 2 +- src/fetch.ts | 18 +++++++++--------- src/helpers/card.ts | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/api/status.ts b/src/api/status.ts index 05a3e5a..ec3475f 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -118,7 +118,7 @@ const populateTweetProperties = async ( /* Populate a Twitter card */ if (tweet.card) { - const card = await renderCard(tweet.card); + const card = renderCard(tweet.card); if (card.external_media) { apiTweet.twitter_card = 'summary_large_image'; apiTweet.media = apiTweet.media || {}; diff --git a/src/fetch.ts b/src/fetch.ts index d66d943..58254c3 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -201,15 +201,15 @@ export const fetchConversation = async ( }/i/api/graphql/TuC3CinYecrqAyqccUyFhw/TweetDetail?variables=${encodeURIComponent( JSON.stringify({ focalTweetId: status, - referrer: 'messages', - with_rux_injections:false, - includePromotedContent:true, - withCommunity:true, - withQuickPromoteEligibilityTweetFields:true, - withArticleRichContent:false, - withBirdwatchNotes:true, - withVoice:true, - withV2Timeline:true + referrer: 'home', + with_rux_injections: false, + includePromotedContent: true, + withCommunity: true, + withQuickPromoteEligibilityTweetFields: true, + withArticleRichContent: false, + withBirdwatchNotes: true, + withVoice: true, + withV2Timeline: true }) )}&features=${encodeURIComponent( JSON.stringify({ diff --git a/src/helpers/card.ts b/src/helpers/card.ts index 1be6ebe..c8e4740 100644 --- a/src/helpers/card.ts +++ b/src/helpers/card.ts @@ -1,9 +1,9 @@ import { calculateTimeLeftString } from './pollTime'; /* Renders card for polls and non-Twitter video embeds (i.e. YouTube) */ -export const renderCard = async ( +export const renderCard = ( card: GraphQLTweet['card'] -): Promise<{ poll?: APIPoll; external_media?: APIExternalMedia }> => { +): { poll?: APIPoll; external_media?: APIExternalMedia } => { // We convert the binding_values array into an object with the legacy format // TODO Clean this up const binding_values: Record = {}; From b790aa70036a959a5f6aca88538c3aa67c63f07d Mon Sep 17 00:00:00 2001 From: Wazbat Date: Wed, 5 Jul 2023 23:12:14 +0200 Subject: [PATCH 5/8] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactored=20to=20use?= =?UTF-8?q?=20different=20tweet=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/status.ts | 62 ++++++++++++------------------ src/api/user.ts | 1 + src/fetch.ts | 75 ++++++++++++++++--------------------- src/helpers/linkFixer.ts | 3 ++ src/types/twitterTypes.d.ts | 12 +++++- test/index.test.ts | 4 +- 6 files changed, 72 insertions(+), 85 deletions(-) diff --git a/src/api/status.ts b/src/api/status.ts index ec3475f..1236b5e 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -189,51 +189,35 @@ export const statusAPI = async ( flags?: InputFlags ): Promise => { let wasMediaBlockedNSFW = false; - let conversation = await fetchConversation(status, event); - let tweet: GraphQLTweet | TweetTombstone; - if (isGraphQLTweetNotFoundResponse(conversation)) { - writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); - return { code: 404, message: 'NOT_FOUND' }; + let res = await fetchConversation(status, event); + const tweet = res.data?.tweetResult?.result; + if (!tweet) { + return { code: 404, message: 'NOT_FOUND' }; } - /* Fallback for if Tweet did not load (i.e. NSFW) */ - if (Object.keys(conversation).length === 0) { - // Try again using elongator API proxy - console.log('No Tweet was found, loading again from elongator'); - conversation = await fetchConversation(status, event, true); - if (Object.keys(conversation).length === 0) { - writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); - return { code: 404, message: 'NOT_FOUND' }; - } - // If the tweet now loads, it was probably NSFW + if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { wasMediaBlockedNSFW = true; + res = await fetchConversation(status, event, true); } - // Find this specific tweet in the conversation - try { - const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions; - if (!Array.isArray(instructions)) { - console.log(JSON.stringify(conversation, null, 2)); - throw new Error('Invalid instructions'); + + if (tweet.__typename === 'TweetUnavailable') { + if (tweet.reason === 'Protected') { + writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags); + return { code: 401, message: 'PRIVATE_TWEET' }; + } else if (tweet.reason === 'NsfwLoggedOut') { + // API failure as elongator should have handled this + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; + } else { + // Api failure at parsing status + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; } - const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries'); - if (!timelineAddEntries) throw new Error('No valid timeline entries'); - const graphQLTimelineTweetEntry = timelineAddEntries.entries - .find((e): e is GraphQLTimelineTweetEntry => - // TODO Fix this idk what's up with the typings - !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`)); - if (!graphQLTimelineTweetEntry) throw new Error('No tweet entry with'); - tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result; - if (!tweet) throw new Error('No tweet in timeline entry'); - } catch (e) { - // Api failure at parsing status - console.log('Tweet could not be accessed, got conversation ', conversation); - writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; } - // If the tweet is not a graphQL tweet it's a tombstone, return the error to the user + // If the tweet is not a graphQL tweet something went wrong if (!isGraphQLTweet(tweet)) { console.log('Tweet was not a valid tweet', tweet); - writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags); - return { code: 401, message: 'PRIVATE_TWEET' }; + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; } /* @@ -245,7 +229,7 @@ export const statusAPI = async ( if (!tweet) { return { code: 404, message: 'NOT_FOUND' }; } - + const conversation: any[] = []; /* Creating the response objects */ const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse; const apiTweet: APITweet = (await populateTweetProperties( diff --git a/src/api/user.ts b/src/api/user.ts index 603d35c..3eca62d 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -14,6 +14,7 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.screen_name = user.legacy.screen_name; apiUser.description = user.legacy.description; apiUser.location = user.legacy.location; + apiUser.banner_url = user.legacy.profile_banner_url; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/fetch.ts b/src/fetch.ts index 58254c3..a08d34e 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,6 +1,6 @@ import { Constants } from './constants'; import { generateUserAgent } from './helpers/useragent'; -import { isGraphQLTweetNotFoundResponse } from './utils/graphql'; +import { isGraphQLTweet, isGraphQLTweetNotFoundResponse } from './utils/graphql'; const API_ATTEMPTS = 16; @@ -59,7 +59,7 @@ export const twitterFetch = async ( '' ); /* Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match */ - const headers: { [header: string]: string } = { + const headers: Record = { Authorization: Constants.GUEST_BEARER_TOKEN, ...Constants.BASE_HEADERS }; @@ -131,12 +131,12 @@ export const twitterFetch = async ( headers: headers }); } - /* - If the tweet is nsfw, the body is empty and status is 404 - const raw = await apiRequest?.clone().text(); - console.log('Raw response:', raw); - console.log('Response code:', apiRequest?.status); - */ + if (apiRequest.status !== 200) { + const raw = await apiRequest?.clone().text(); + console.log('Raw response:', raw); + console.log('Response code:', apiRequest?.status); + } + response = await apiRequest?.json(); } catch (e: unknown) { /* We'll usually only hit this if we get an invalid response from Twitter. @@ -194,31 +194,15 @@ export const fetchConversation = async ( status: string, event: FetchEvent, useElongator = false -): Promise => { +): Promise => { return (await twitterFetch( `${ Constants.TWITTER_ROOT - }/i/api/graphql/TuC3CinYecrqAyqccUyFhw/TweetDetail?variables=${encodeURIComponent( - JSON.stringify({ - focalTweetId: status, - referrer: 'home', - with_rux_injections: false, - includePromotedContent: true, - withCommunity: true, - withQuickPromoteEligibilityTweetFields: true, - withArticleRichContent: false, - withBirdwatchNotes: true, - withVoice: true, - withV2Timeline: true - }) + }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( + JSON.stringify({"tweetId": status,"withCommunity":false,"includePromotedContent":false,"withVoice":false}) )}&features=${encodeURIComponent( JSON.stringify({ - rweb_lists_timeline_redesign_enabled:true, - responsive_web_graphql_exclude_directive_enabled:true, - verified_phone_label_enabled:false, creator_subscriptions_tweet_preview_api_enabled:true, - responsive_web_graphql_timeline_navigation_enabled:true, - responsive_web_graphql_skip_user_profile_image_extensions_enabled:false, tweetypie_unmention_optimization_enabled:true, responsive_web_edit_tweet_api_enabled:true, graphql_is_translatable_rweb_tweet_is_translatable_enabled:true, @@ -231,9 +215,12 @@ export const fetchConversation = async ( tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:true, longform_notetweets_rich_text_read_enabled:true, longform_notetweets_inline_media_enabled:true, + responsive_web_graphql_exclude_directive_enabled:true, + verified_phone_label_enabled:false, responsive_web_media_download_video_enabled:false, - responsive_web_enhance_cards_enabled:false - }) + responsive_web_graphql_skip_user_profile_image_extensions_enabled:false, + responsive_web_graphql_timeline_navigation_enabled:true, + responsive_web_enhance_cards_enabled:false}) )}&fieldToggles=${encodeURIComponent( JSON.stringify({ // TODO Figure out what this property does @@ -243,22 +230,24 @@ export const fetchConversation = async ( event, useElongator, (_conversation: unknown) => { - const conversation = _conversation as GraphQLTweetDetailResponse; + const conversation = _conversation as TweetResultsByRestIdResult; // If we get a not found error it's still a valid response - if (isGraphQLTweetNotFoundResponse(conversation)) return true; - const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions; - if (!Array.isArray(instructions)) return false; - const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries'); - if (!timelineAddEntries) return false; - const graphQLTimelineTweetEntry = timelineAddEntries.entries - .find((e): e is GraphQLTimelineTweetEntry => - // TODO Fix this idk what's up with the typings - !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`)); - if (!graphQLTimelineTweetEntry) return false; - const tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result; - return !!tweet; + const tweet = conversation.data?.tweetResult?.result; + if (isGraphQLTweet(tweet)) { + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { + return true; + } + if (Array.isArray(conversation.errors)) { + return true; + } + return false; } - )) as GraphQLTweetDetailResponse; + )) as TweetResultsByRestIdResult; }; export const fetchUser = async ( diff --git a/src/helpers/linkFixer.ts b/src/helpers/linkFixer.ts index 29a82d3..825494b 100644 --- a/src/helpers/linkFixer.ts +++ b/src/helpers/linkFixer.ts @@ -1,5 +1,8 @@ /* Helps replace t.co links with their originals */ export const linkFixer = (tweet: GraphQLTweet, text: string): string => { + console.log('got entites', { + entities: tweet.legacy.entities, + }) if (Array.isArray(tweet.legacy.entities?.urls) && tweet.legacy.entities.urls.length) { tweet.legacy.entities.urls.forEach((url: TcoExpansion) => { let newURL = url.expanded_url; diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index bf85f94..d71bdde 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -456,4 +456,14 @@ type GraphQLTweetFoundResponse = { } } } -type GraphQLTweetDetailResponse = GraphQLTweetFoundResponse | GraphQLTweetNotFoundResponse; \ No newline at end of file +type TweetResultsByRestIdResult = { + errors?: unknown[]; + data?: { + tweetResult?: { + result?: { + __typename: 'TweetUnavailable'; + reason: 'NsfwLoggedOut'|'Protected'; + }|GraphQLTweet + } + } +} \ No newline at end of file diff --git a/test/index.test.ts b/test/index.test.ts index cccd662..52ea87b 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -115,7 +115,7 @@ test('API fetch video Tweet', async () => { expect(tweet.url).toEqual('https://twitter.com/Twitter/status/854416760933556224'); expect(tweet.id).toEqual('854416760933556224'); expect(tweet.text).toEqual( - 'Get the sauces ready, #NuggsForCarter has 3 million+ Retweets.' + 'Get the sauces ready, #NuggsForCarter has 3 million+ Retweets. https://t.co/ydLBtfK3Z3' ); expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter'); expect(tweet.author.id).toEqual('783214'); @@ -160,7 +160,7 @@ test('API fetch multi-photo Tweet', async () => { expect(tweet).toBeTruthy(); expect(tweet.url).toEqual('https://twitter.com/Twitter/status/1445094085593866246'); expect(tweet.id).toEqual('1445094085593866246'); - expect(tweet.text).toEqual('@netflix'); + expect(tweet.text).toEqual('@netflix https://t.co/W0XPnj2qLP'); expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter'); expect(tweet.author.id).toEqual('783214'); expect(tweet.author.name).toBeTruthy(); From 3b6759566b04936672904131635f237945f72900 Mon Sep 17 00:00:00 2001 From: Wazbat Date: Wed, 5 Jul 2023 23:27:49 +0200 Subject: [PATCH 6/8] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactored=20fetch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fetch.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index a08d34e..921b143 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,6 +1,6 @@ import { Constants } from './constants'; import { generateUserAgent } from './helpers/useragent'; -import { isGraphQLTweet, isGraphQLTweetNotFoundResponse } from './utils/graphql'; +import { isGraphQLTweet } from './utils/graphql'; const API_ATTEMPTS = 16; @@ -242,10 +242,11 @@ export const fetchConversation = async ( if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { return true; } - if (Array.isArray(conversation.errors)) { + if (tweet?.__typename === 'TweetUnavailable') { return true; } - return false; + // Final clause for checking if it's valid is if there's errors + return Array.isArray(conversation.errors) } )) as TweetResultsByRestIdResult; }; From f561205d7a55b5ec1655a18ca56daca37863d697 Mon Sep 17 00:00:00 2001 From: Wazbat Date: Wed, 5 Jul 2023 23:29:10 +0200 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=94=87=20Removed=20log=20used=20durin?= =?UTF-8?q?g=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fetch.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index 921b143..6077586 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -131,11 +131,6 @@ export const twitterFetch = async ( headers: headers }); } - if (apiRequest.status !== 200) { - const raw = await apiRequest?.clone().text(); - console.log('Raw response:', raw); - console.log('Response code:', apiRequest?.status); - } response = await apiRequest?.json(); } catch (e: unknown) { From d6c1b6a82af8100ff2b1f90cba6228051f8b7400 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Mon, 14 Aug 2023 21:01:13 -0400 Subject: [PATCH 8/8] Fix some recent api woes (still not complete) --- motd.json | 3 +-- src/api/status.ts | 2 ++ src/constants.ts | 2 +- src/embed/status.ts | 1 + src/fetch.ts | 17 ++++++++++------- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/motd.json b/motd.json index 344f288..7c504e4 100644 --- a/motd.json +++ b/motd.json @@ -1,4 +1,3 @@ { - "FixTweet": "https://github.com/FixTweet/FixTweet", - "FixTweet - Embed videos, polls & more": "https://github.com/FixTweet/FixTweet" + "FixTweet - Recovering from API woes": "https://github.com/FixTweet/FixTweet/issues/333" } diff --git a/src/api/status.ts b/src/api/status.ts index 1236b5e..dd785be 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -198,6 +198,8 @@ export const statusAPI = async ( wasMediaBlockedNSFW = true; res = await fetchConversation(status, event, true); } + + console.log(JSON.stringify(tweet)) if (tweet.__typename === 'TweetUnavailable') { if (tweet.reason === 'Protected') { diff --git a/src/constants.ts b/src/constants.ts index 45be395..9072ae1 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -20,7 +20,7 @@ export const Constants = { GUEST_TOKEN_MAX_AGE: 3 * 60 * 60, /* Twitter Web App actually uses Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA instead, but accounts marked as 18+ wouldn't show up then */ - GUEST_BEARER_TOKEN: `Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw`, + GUEST_BEARER_TOKEN: `Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA`, GUEST_FETCH_PARAMETERS: [ 'cards_platform=Web-12', 'include_cards=1', diff --git a/src/embed/status.ts b/src/embed/status.ts index 16caa1c..1a34ca9 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -59,6 +59,7 @@ export const handleStatus = async ( case 404: return returnError(Strings.ERROR_TWEET_NOT_FOUND); case 500: + console.log(api); return returnError(Strings.ERROR_API_FAIL); } diff --git a/src/fetch.ts b/src/fetch.ts index 921b143..0f8bc95 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -2,7 +2,13 @@ import { Constants } from './constants'; import { generateUserAgent } from './helpers/useragent'; import { isGraphQLTweet } from './utils/graphql'; -const API_ATTEMPTS = 16; +const API_ATTEMPTS = 3; + +function generateCSRFToken() { + const randomBytes = new Uint8Array(160/2); + crypto.getRandomValues(randomBytes); + return Array.from(randomBytes, byte => byte.toString(16).padStart(2, '0')).join(''); +} export const twitterFetch = async ( url: string, @@ -52,12 +58,9 @@ export const twitterFetch = async ( const cache = caches.default; while (apiAttempts < API_ATTEMPTS) { - const csrfToken = crypto - .randomUUID() - .replace( - /-/g, - '' - ); /* Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match */ + /* Generate a random CSRF token, Twitter just cares that header and cookie match, + REST can use shorter csrf tokens (32 bytes) but graphql prefers 160 bytes */ + const csrfToken = generateCSRFToken(); const headers: Record = { Authorization: Constants.GUEST_BEARER_TOKEN,