♻️ Refactored to use different tweet endpoint

src/api/status.ts

@@ -189,51 +189,35 @@ export const statusAPI = async (
   flags?: InputFlags
 ): Promise<TweetAPIResponse> => {
   let wasMediaBlockedNSFW = false;
-  let conversation = await fetchConversation(status, event);
+  let res = await fetchConversation(status, event);
-  let tweet: GraphQLTweet | TweetTombstone;
+  const tweet = res.data?.tweetResult?.result;
-  if (isGraphQLTweetNotFoundResponse(conversation)) { 
+  if (!tweet) {
-      writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags);
+    return { code: 404, message: 'NOT_FOUND' };
-      return { code: 404, message: 'NOT_FOUND' };
   }
-  /* Fallback for if Tweet did not load (i.e. NSFW) */
+  if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
-  if (Object.keys(conversation).length === 0) {
-    // Try again using elongator API proxy
-    console.log('No Tweet was found, loading again from elongator');
-    conversation = await fetchConversation(status, event, true);
-    if (Object.keys(conversation).length === 0) {
-      writeDataPoint(event, language, wasMediaBlockedNSFW, 'NOT_FOUND', flags);
-      return { code: 404, message: 'NOT_FOUND' };
-    }
-    // If the tweet now loads, it was probably NSFW
     wasMediaBlockedNSFW = true;
+    res = await fetchConversation(status, event, true);
   }
-  // Find this specific tweet in the conversation
+  
-  try {
+  if (tweet.__typename === 'TweetUnavailable') {
-    const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions;
+    if (tweet.reason === 'Protected') {
-    if (!Array.isArray(instructions)) {
+      writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags);
-      console.log(JSON.stringify(conversation, null, 2));
+      return { code: 401, message: 'PRIVATE_TWEET' };
-      throw new Error('Invalid instructions');
+    } else if (tweet.reason === 'NsfwLoggedOut') {
+      // API failure as elongator should have handled this
+      writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags);
+      return { code: 500, message: 'API_FAIL' };
+    } else {
+      // Api failure at parsing status
+      writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags);
+      return { code: 500, message: 'API_FAIL' };
     }
-    const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries');
-    if (!timelineAddEntries) throw new Error('No valid timeline entries');
-    const graphQLTimelineTweetEntry = timelineAddEntries.entries
-    .find((e): e is GraphQLTimelineTweetEntry =>
-    // TODO Fix this idk what's up with the typings
-    !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`));
-    if (!graphQLTimelineTweetEntry) throw new Error('No tweet entry with');
-    tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result;
-    if (!tweet) throw new Error('No tweet in timeline entry');
-  } catch (e) {
-    // Api failure at parsing status
-    console.log('Tweet could not be accessed, got conversation ', conversation);
-    writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags);
-    return { code: 500, message: 'API_FAIL' };
   }
-  // If the tweet is not a graphQL tweet it's a tombstone, return the error to the user
+  // If the tweet is not a graphQL tweet something went wrong
   if (!isGraphQLTweet(tweet)) {
     console.log('Tweet was not a valid tweet', tweet);
-    writeDataPoint(event, language, wasMediaBlockedNSFW, 'PRIVATE_TWEET', flags);
+    writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags);
-    return { code: 401, message: 'PRIVATE_TWEET' };
+    return { code: 500, message: 'API_FAIL' };
   }
   
   /*
@@ -245,7 +229,7 @@ export const statusAPI = async (
   if (!tweet) {
     return { code: 404, message: 'NOT_FOUND' };
   }
-
+  const conversation: any[] = [];
   /* Creating the response objects */
   const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse;
   const apiTweet: APITweet = (await populateTweetProperties(

src/api/user.ts

@@ -14,6 +14,7 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
   apiUser.screen_name = user.legacy.screen_name;
   apiUser.description = user.legacy.description;
   apiUser.location = user.legacy.location;
+  apiUser.banner_url = user.legacy.profile_banner_url;
   /*
   if (user.is_blue_verified) {
     apiUser.verified = 'blue';

src/fetch.ts

@@ -1,6 +1,6 @@
 import { Constants } from './constants';
 import { generateUserAgent } from './helpers/useragent';
-import { isGraphQLTweetNotFoundResponse } from './utils/graphql';
+import { isGraphQLTweet, isGraphQLTweetNotFoundResponse } from './utils/graphql';
 
 const API_ATTEMPTS = 16;
 
@@ -59,7 +59,7 @@ export const twitterFetch = async (
         ''
       ); /* Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match */
 
-    const headers: { [header: string]: string } = {
+    const headers: Record<string, string> = {
       Authorization: Constants.GUEST_BEARER_TOKEN,
       ...Constants.BASE_HEADERS
     };
@@ -131,12 +131,12 @@ export const twitterFetch = async (
           headers: headers
         });
       }
-      /*
+      if (apiRequest.status !== 200) {
-      If the tweet is nsfw, the body is empty and status is 404
+        const raw = await apiRequest?.clone().text();
-      const raw = await apiRequest?.clone().text();
+        console.log('Raw response:', raw);
-      console.log('Raw response:', raw);
+        console.log('Response code:', apiRequest?.status);
-      console.log('Response code:', apiRequest?.status);
+      }
-      */
+      
       response = await apiRequest?.json();
     } catch (e: unknown) {
       /* We'll usually only hit this if we get an invalid response from Twitter.
@@ -194,31 +194,15 @@ export const fetchConversation = async (
   status: string,
   event: FetchEvent,
   useElongator = false
-): Promise<GraphQLTweetDetailResponse> => {
+): Promise<TweetResultsByRestIdResult> => {
   return (await twitterFetch(
     `${
       Constants.TWITTER_ROOT
-    }/i/api/graphql/TuC3CinYecrqAyqccUyFhw/TweetDetail?variables=${encodeURIComponent(
+    }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent(
-      JSON.stringify({
+      JSON.stringify({"tweetId": status,"withCommunity":false,"includePromotedContent":false,"withVoice":false})
-        focalTweetId: status,
-        referrer: 'home',
-        with_rux_injections: false,
-        includePromotedContent: true,
-        withCommunity: true,
-        withQuickPromoteEligibilityTweetFields: true,
-        withArticleRichContent: false,
-        withBirdwatchNotes: true,
-        withVoice: true,
-        withV2Timeline: true
-      })
     )}&features=${encodeURIComponent(
       JSON.stringify({
-        rweb_lists_timeline_redesign_enabled:true,
-        responsive_web_graphql_exclude_directive_enabled:true,
-        verified_phone_label_enabled:false,
         creator_subscriptions_tweet_preview_api_enabled:true,
-        responsive_web_graphql_timeline_navigation_enabled:true,
-        responsive_web_graphql_skip_user_profile_image_extensions_enabled:false,
         tweetypie_unmention_optimization_enabled:true,
         responsive_web_edit_tweet_api_enabled:true,
         graphql_is_translatable_rweb_tweet_is_translatable_enabled:true,
@@ -231,9 +215,12 @@ export const fetchConversation = async (
         tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:true,
         longform_notetweets_rich_text_read_enabled:true,
         longform_notetweets_inline_media_enabled:true,
+        responsive_web_graphql_exclude_directive_enabled:true,
+        verified_phone_label_enabled:false,
         responsive_web_media_download_video_enabled:false,
-        responsive_web_enhance_cards_enabled:false
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled:false,
-      })
+        responsive_web_graphql_timeline_navigation_enabled:true,
+        responsive_web_enhance_cards_enabled:false})
     )}&fieldToggles=${encodeURIComponent(
       JSON.stringify({
         // TODO Figure out what this property does
@@ -243,22 +230,24 @@ export const fetchConversation = async (
     event,
     useElongator,
     (_conversation: unknown) => {
-      const conversation = _conversation as GraphQLTweetDetailResponse;
+      const conversation = _conversation as TweetResultsByRestIdResult;
       // If we get a not found error it's still a valid response
-      if (isGraphQLTweetNotFoundResponse(conversation)) return true;
+      const tweet = conversation.data?.tweetResult?.result;
-      const instructions = conversation?.data?.threaded_conversation_with_injections_v2?.instructions;
+      if (isGraphQLTweet(tweet)) {
-      if (!Array.isArray(instructions)) return false;
+        return true;
-      const timelineAddEntries = instructions.find((e): e is TimeLineAddEntriesInstruction => e?.type === 'TimelineAddEntries');
+      }
-      if (!timelineAddEntries) return false;
+      if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
-      const graphQLTimelineTweetEntry = timelineAddEntries.entries
+        return true;
-      .find((e): e is GraphQLTimelineTweetEntry =>
+      }
-      // TODO Fix this idk what's up with the typings
+      if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
-      !!(e && typeof e === 'object' && ('entryId' in e) && e?.entryId === `tweet-${status}`));
+        return true;
-      if (!graphQLTimelineTweetEntry) return false;
+      }
-      const tweet = graphQLTimelineTweetEntry?.content?.itemContent?.tweet_results?.result;
+      if (Array.isArray(conversation.errors)) {
-      return !!tweet;
+        return true;
+      }
+      return false;
     }
-  )) as GraphQLTweetDetailResponse;
+  )) as TweetResultsByRestIdResult;
 };
 
 export const fetchUser = async (

src/helpers/linkFixer.ts

@@ -1,5 +1,8 @@
 /* Helps replace t.co links with their originals */
 export const linkFixer = (tweet: GraphQLTweet, text: string): string => {
+  console.log('got entites', {
+    entities: tweet.legacy.entities,
+  })
   if (Array.isArray(tweet.legacy.entities?.urls) && tweet.legacy.entities.urls.length) {
     tweet.legacy.entities.urls.forEach((url: TcoExpansion) => {
       let newURL = url.expanded_url;

src/types/twitterTypes.d.ts

@@ -456,4 +456,14 @@ type GraphQLTweetFoundResponse = {
     }
   }
 }
-type GraphQLTweetDetailResponse = GraphQLTweetFoundResponse | GraphQLTweetNotFoundResponse;
+type TweetResultsByRestIdResult = {
+  errors?: unknown[];
+  data?: {
+    tweetResult?: {
+      result?: {
+        __typename: 'TweetUnavailable';
+        reason: 'NsfwLoggedOut'|'Protected';
+      }|GraphQLTweet
+    }
+  }
+}

test/index.test.ts

@@ -115,7 +115,7 @@ test('API fetch video Tweet', async () => {
   expect(tweet.url).toEqual('https://twitter.com/Twitter/status/854416760933556224');
   expect(tweet.id).toEqual('854416760933556224');
   expect(tweet.text).toEqual(
-    'Get the sauces ready, #NuggsForCarter has 3 million+ Retweets.'
+    'Get the sauces ready, #NuggsForCarter has 3 million+ Retweets. https://t.co/ydLBtfK3Z3'
   );
   expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter');
   expect(tweet.author.id).toEqual('783214');
@@ -160,7 +160,7 @@ test('API fetch multi-photo Tweet', async () => {
   expect(tweet).toBeTruthy();
   expect(tweet.url).toEqual('https://twitter.com/Twitter/status/1445094085593866246');
   expect(tweet.id).toEqual('1445094085593866246');
-  expect(tweet.text).toEqual('@netflix');
+  expect(tweet.text).toEqual('@netflix https://t.co/W0XPnj2qLP');
   expect(tweet.author.screen_name?.toLowerCase()).toEqual('twitter');
   expect(tweet.author.id).toEqual('783214');
   expect(tweet.author.name).toBeTruthy();