diff --git a/src/api/status.ts b/src/api/status.ts index ec3475f..1236b5e 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -189,51 +189,35 @@ export const statusAPI = async ( flags?: InputFlags ): Promise => { let wasMediaBlockedNSFW = false; - let conversation = await fetchConversation(status, event); - let tweet: GraphQLTweet | TweetTombstone; - if (isGraphQLTweetNotFoundResponse(conversation)) { - writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); - return { code: 404, message: 'NOT_FOUND' }; + let res = await fetchConversation(status, event); + const tweet = res.data?.tweetResult?.result; + if (!tweet) { + return { code: 404, message: 'NOT_FOUND' }; } - /* Fallback for if Tweet did not load (i.e. NSFW) */ - if (Object.keys(conversation).length === 0) { - // Try again using elongator API proxy - console.log('No Tweet was found, loading again from elongator'); - conversation = await fetchConversation(status, event, true); - if (Object.keys(conversation).length === 0) { - writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags); - return { code: 404, message: 'NOT_FOUND' }; - } - // If the tweet now loads, it was probably NSFW + if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { wasMediaBlockedNSFW = true; + res = await fetchConversation(status, event, true); } - // Find this specific tweet in the conversation - try { - const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions; - if (!Array.isArray(instructions)) { - console.log(JSON.stringify(conversation, null, 2)); - throw new Error('Invalid instructions'); + + if (tweet.__typename === 'TweetUnavailable') { + if (tweet.reason === 'Protected') { + writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags); + return { code: 401, message: 'PRIVATE_TWEET' }; + } else if (tweet.reason === 'NsfwLoggedOut') { + // API failure as elongator should have handled this + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; + } else { + // Api failure at parsing status + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; } - const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries'); - if (!timelineAddEntries) throw new Error('No valid timeline entries'); - const graphQLTimelineTweetEntry = timelineAddEntries.entries - .find((e): e is GraphQLTimelineTweetEntry => - // TODO Fix this idk what's up with the typings - !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`)); - if (!graphQLTimelineTweetEntry) throw new Error('No tweet entry with'); - tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result; - if (!tweet) throw new Error('No tweet in timeline entry'); - } catch (e) { - // Api failure at parsing status - console.log('Tweet could not be accessed, got conversation ', conversation); - writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; } - // If the tweet is not a graphQL tweet it's a tombstone, return the error to the user + // If the tweet is not a graphQL tweet something went wrong if (!isGraphQLTweet(tweet)) { console.log('Tweet was not a valid tweet', tweet); - writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags); - return { code: 401, message: 'PRIVATE_TWEET' }; + writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); + return { code: 500, message: 'API_FAIL' }; } /* @@ -245,7 +229,7 @@ export const statusAPI = async ( if (!tweet) { return { code: 404, message: 'NOT_FOUND' }; } - + const conversation: any[] = []; /* Creating the response objects */ const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse; const apiTweet: APITweet = (await populateTweetProperties( diff --git a/src/api/user.ts b/src/api/user.ts index 603d35c..3eca62d 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -14,6 +14,7 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.screen_name = user.legacy.screen_name; apiUser.description = user.legacy.description; apiUser.location = user.legacy.location; + apiUser.banner_url = user.legacy.profile_banner_url; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/fetch.ts b/src/fetch.ts index 58254c3..a08d34e 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,6 +1,6 @@ import { Constants } from './constants'; import { generateUserAgent } from './helpers/useragent'; -import { isGraphQLTweetNotFoundResponse } from './utils/graphql'; +import { isGraphQLTweet, isGraphQLTweetNotFoundResponse } from './utils/graphql'; const API_ATTEMPTS = 16; @@ -59,7 +59,7 @@ export const twitterFetch = async ( '' ); /* Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match */ - const headers: { [header: string]: string } = { + const headers: Record = { Authorization: Constants.GUEST_BEARER_TOKEN, ...Constants.BASE_HEADERS }; @@ -131,12 +131,12 @@ export const twitterFetch = async ( headers: headers }); } - /* - If the tweet is nsfw, the body is empty and status is 404 - const raw = await apiRequest?.clone().text(); - console.log('Raw response:', raw); - console.log('Response code:', apiRequest?.status); - */ + if (apiRequest.status !== 200) { + const raw = await apiRequest?.clone().text(); + console.log('Raw response:', raw); + console.log('Response code:', apiRequest?.status); + } + response = await apiRequest?.json(); } catch (e: unknown) { /* We'll usually only hit this if we get an invalid response from Twitter. @@ -194,31 +194,15 @@ export const fetchConversation = async ( status: string, event: FetchEvent, useElongator = false -): Promise => { +): Promise => { return (await twitterFetch( `${ Constants.TWITTER_ROOT - }/i/api/graphql/TuC3CinYecrqAyqccUyFhw/TweetDetail?variables=${encodeURIComponent( - JSON.stringify({ - focalTweetId: status, - referrer: 'home', - with_rux_injections: false, - includePromotedContent: true, - withCommunity: true, - withQuickPromoteEligibilityTweetFields: true, - withArticleRichContent: false, - withBirdwatchNotes: true, - withVoice: true, - withV2Timeline: true - }) + }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( + JSON.stringify({"tweetId": status,"withCommunity":false,"includePromotedContent":false,"withVoice":false}) )}&features=${encodeURIComponent( JSON.stringify({ - rweb_lists_timeline_redesign_enabled:true, - responsive_web_graphql_exclude_directive_enabled:true, - verified_phone_label_enabled:false, creator_subscriptions_tweet_preview_api_enabled:true, - responsive_web_graphql_timeline_navigation_enabled:true, - responsive_web_graphql_skip_user_profile_image_extensions_enabled:false, tweetypie_unmention_optimization_enabled:true, responsive_web_edit_tweet_api_enabled:true, graphql_is_translatable_rweb_tweet_is_translatable_enabled:true, @@ -231,9 +215,12 @@ export const fetchConversation = async ( tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:true, longform_notetweets_rich_text_read_enabled:true, longform_notetweets_inline_media_enabled:true, + responsive_web_graphql_exclude_directive_enabled:true, + verified_phone_label_enabled:false, responsive_web_media_download_video_enabled:false, - responsive_web_enhance_cards_enabled:false - }) + responsive_web_graphql_skip_user_profile_image_extensions_enabled:false, + responsive_web_graphql_timeline_navigation_enabled:true, + responsive_web_enhance_cards_enabled:false}) )}&fieldToggles=${encodeURIComponent( JSON.stringify({ // TODO Figure out what this property does @@ -243,22 +230,24 @@ export const fetchConversation = async ( event, useElongator, (_conversation: unknown) => { - const conversation = _conversation as GraphQLTweetDetailResponse; + const conversation = _conversation as TweetResultsByRestIdResult; // If we get a not found error it's still a valid response - if (isGraphQLTweetNotFoundResponse(conversation)) return true; - const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions; - if (!Array.isArray(instructions)) return false; - const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries'); - if (!timelineAddEntries) return false; - const graphQLTimelineTweetEntry = timelineAddEntries.entries - .find((e): e is GraphQLTimelineTweetEntry => - // TODO Fix this idk what's up with the typings - !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`)); - if (!graphQLTimelineTweetEntry) return false; - const tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result; - return !!tweet; + const tweet = conversation.data?.tweetResult?.result; + if (isGraphQLTweet(tweet)) { + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { + return true; + } + if (Array.isArray(conversation.errors)) { + return true; + } + return false; } - )) as GraphQLTweetDetailResponse; + )) as TweetResultsByRestIdResult; }; export const fetchUser = async ( diff --git a/src/helpers/linkFixer.ts b/src/helpers/linkFixer.ts index 29a82d3..825494b 100644 --- a/src/helpers/linkFixer.ts +++ b/src/helpers/linkFixer.ts @@ -1,5 +1,8 @@ /* Helps replace t.co links with their originals */ export const linkFixer = (tweet: GraphQLTweet, text: string): string => { + console.log('got entites', { + entities: tweet.legacy.entities, + }) if (Array.isArray(tweet.legacy.entities?.urls) && tweet.legacy.entities.urls.length) { tweet.legacy.entities.urls.forEach((url: TcoExpansion) => { let newURL = url.expanded_url; diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index bf85f94..d71bdde 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -456,4 +456,14 @@ type GraphQLTweetFoundResponse = { } } } -type GraphQLTweetDetailResponse = GraphQLTweetFoundResponse | GraphQLTweetNotFoundResponse; \ No newline at end of file +type TweetResultsByRestIdResult = { + errors?: unknown[]; + data?: { + tweetResult?: { + result?: { + __typename: 'TweetUnavailable'; + reason: 'NsfwLoggedOut'|'Protected'; + }|GraphQLTweet + } + } +} \ No newline at end of file diff --git a/test/index.test.ts b/test/index.test.ts index cccd662..52ea87b 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -115,7 +115,7 @@ test('API fetch video Tweet', async () => { expect(tweet.url).toEqual('https://twitter.com/Twitter/status/854416760933556224'); expect(tweet.id).toEqual('854416760933556224'); expect(tweet.text).toEqual( - 'Get the sauces ready, #NuggsForCarter has 3 million+ Retweets.' + 'Get the sauces ready, #NuggsForCarter has 3 million+ Retweets. https://t.co/ydLBtfK3Z3' ); expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter'); expect(tweet.author.id).toEqual('783214'); @@ -160,7 +160,7 @@ test('API fetch multi-photo Tweet', async () => { expect(tweet).toBeTruthy(); expect(tweet.url).toEqual('https://twitter.com/Twitter/status/1445094085593866246'); expect(tweet.id).toEqual('1445094085593866246'); - expect(tweet.text).toEqual('@netflix'); + expect(tweet.text).toEqual('@netflix https://t.co/W0XPnj2qLP'); expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter'); expect(tweet.author.id).toEqual('783214'); expect(tweet.author.name).toBeTruthy();