WIP embed rework

src/api/status.ts

@@ -77,7 +77,7 @@ const populateTweetProperties = async (
     website: apiUser.website
   };
   apiTweet.replies = tweet.legacy.reply_count;
-  apiTweet.retweets = tweet.legacy.retweet_count;
+  apiTweet.reposts = tweet.legacy.retweet_count;
   apiTweet.reposts = tweet.legacy.retweet_count;
   apiTweet.likes = tweet.legacy.favorite_count;
   // @ts-expect-error Legacy api shit 

src/embed/status.ts

@@ -3,10 +3,11 @@ import { handleQuote } from '../helpers/quote';
 import { formatNumber, sanitizeText, truncateWithEllipsis } from '../helpers/utils';
 import { Strings } from '../strings';
 import { getAuthorText } from '../helpers/author';
-import { statusAPI } from '../api/status';
 import { renderPhoto } from '../render/photo';
 import { renderVideo } from '../render/video';
 import { renderInstantView } from '../render/instantview';
+import { constructTwitterThread } from '../providers/twitter/conversation';
+import { IRequest } from 'itty-router';
 
 export const returnError = (error: string): StatusResponse => {
   return {
@@ -33,8 +34,38 @@ export const handleStatus = async (
 ): Promise<StatusResponse> => {
   console.log('Direct?', flags?.direct);
 
-  const api = await statusAPI(status, language, event as FetchEvent, flags);
-  const tweet = api?.tweet as APITweet;
+  const request = (event as FetchEvent).request as IRequest;
+  let fetchWithThreads = false;
+
+  if (request.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) {
+    fetchWithThreads = true;
+  }
+
+  const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined);
+
+  const tweet = thread?.post as APITweet;
+
+  const api = {
+    code: thread.code,
+    message: '',
+    tweet: tweet
+  };
+
+  switch(api.code) {
+    case 200:
+      api.message = "OK";
+      break;
+    case 401:
+      api.message = "PRIVATE_TWEET";
+      break;
+    case 404:
+      api.message = "NOT_FOUND";
+      break;
+    case 500:
+      console.log(api);
+      api.message = "API_FAIL";
+      break;
+  }
 
   /* Catch this request if it's an API response */
   if (flags?.api) {
@@ -46,6 +77,10 @@ export const handleStatus = async (
     };
   }
 
+  if (tweet === null) {
+    return returnError(Strings.ERROR_TWEET_NOT_FOUND);
+  }
+
   /* If there was any errors fetching the Tweet, we'll return it */
   switch (api.code) {
     case 401:

src/experiments.ts

@@ -1,6 +1,7 @@
 export enum Experiment {
   ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT',
-  ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API'
+  ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API',
+  TWEET_DETAIL_API = 'TWEET_DETAIL_API',
 }
 
 type ExperimentConfig = {
@@ -19,7 +20,12 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = {
     name: 'Elongator profile API',
     description: 'Use Elongator to load profiles',
     percentage: 0
-  }
+  },
+  [Experiment.TWEET_DETAIL_API]: {
+    name: 'Tweet detail API',
+    description: 'Use Tweet Detail API (where available with elongator)',
+    percentage: 0.75
+  },
 };
 
 export const experimentCheck = (experiment: Experiment, condition = true) => {

src/helpers/author.ts

@@ -3,13 +3,13 @@ import { formatNumber } from './utils';
 /* The embed "author" text we populate with replies, retweets, and likes unless it's a video */
 export const getAuthorText = (tweet: APITweet): string | null => {
   /* Build out reply, retweet, like counts */
-  if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) {
+  if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) {
     let authorText = '';
     if (tweet.replies > 0) {
       authorText += `${formatNumber(tweet.replies)} 💬    `;
     }
-    if (tweet.retweets > 0) {
-      authorText += `${formatNumber(tweet.retweets)} 🔁    `;
+    if (tweet.reposts > 0) {
+      authorText += `${formatNumber(tweet.reposts)} 🔁    `;
     }
     if (tweet.likes > 0) {
       authorText += `${formatNumber(tweet.likes)} ❤️    `;
@@ -28,13 +28,13 @@ export const getAuthorText = (tweet: APITweet): string | null => {
 /* The embed "author" text we populate with replies, retweets, and likes unless it's a video */
 export const getSocialTextIV = (tweet: APITweet): string | null => {
   /* Build out reply, retweet, like counts */
-  if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) {
+  if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) {
     let authorText = '';
     if (tweet.replies > 0) {
       authorText += `💬 ${formatNumber(tweet.replies)} `;
     }
-    if (tweet.retweets > 0) {
-      authorText += `🔁 ${formatNumber(tweet.retweets)} `;
+    if (tweet.reposts > 0) {
+      authorText += `🔁 ${formatNumber(tweet.reposts)} `;
     }
     if (tweet.likes > 0) {
       authorText += `❤️ ${formatNumber(tweet.likes)} `;

src/providers/twitter/conversation.ts

@@ -2,19 +2,10 @@ import { IRequest } from "itty-router";
 import { Constants } from "../../constants";
 import { twitterFetch } from "../../fetch";
 import { buildAPITweet } from "./processor";
+import { Experiment, experimentCheck } from "../../experiments";
+import { isGraphQLTweet } from "../../helpers/graphql";
 
-type GraphQLProcessBucket = {
-  tweets: GraphQLTweet[];
-  cursors: GraphQLTimelineCursor[];
-}
-
-type SocialThread = {
-  post: APIPost | APITweet | null;
-  thread: (APIPost | APITweet)[] | null;
-  author: APIUser | null;
-}
-
-export const fetchTwitterThread = async (
+export const fetchTweetDetail = async (
   status: string,
   event: FetchEvent,
   useElongator = typeof TwitterProxy !== 'undefined',
@@ -66,13 +57,102 @@ export const fetchTwitterThread = async (
     )}`,
     event,
     useElongator,
-    () => {
+    (_conversation: unknown) => {
+      const conversation = _conversation as GraphQLTweetFoundResponse;
+      const tweet = findTweetInBucket(status, processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions));
+      if (tweet && isGraphQLTweet(tweet)) {
         return true;
+      }
+      console.log('invalid graphql tweet');
+
+      return Array.isArray(conversation?.errors);
     }
   )) as GraphQLTweetFoundResponse;
 };
 
-const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => {
+export const fetchByRestId = async (
+  status: string,
+  event: FetchEvent,
+  useElongator = experimentCheck(
+    Experiment.ELONGATOR_BY_DEFAULT,
+    typeof TwitterProxy !== 'undefined'
+  )
+): Promise<TweetResultsByRestIdResult> => {
+  return (await twitterFetch(
+    `${
+      Constants.TWITTER_ROOT
+    }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent(
+      JSON.stringify({
+        tweetId: status,
+        withCommunity: false,
+        includePromotedContent: false,
+        withVoice: false
+      })
+    )}&features=${encodeURIComponent(
+      JSON.stringify({
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        tweetypie_unmention_optimization_enabled: true,
+        responsive_web_edit_tweet_api_enabled: true,
+        graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+        view_counts_everywhere_api_enabled: true,
+        longform_notetweets_consumption_enabled: true,
+        responsive_web_twitter_article_tweet_consumption_enabled: false,
+        tweet_awards_web_tipping_enabled: false,
+        freedom_of_speech_not_reach_fetch_enabled: true,
+        standardized_nudges_misinfo: true,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        longform_notetweets_rich_text_read_enabled: true,
+        longform_notetweets_inline_media_enabled: true,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        responsive_web_media_download_video_enabled: false,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+        responsive_web_enhance_cards_enabled: false
+      })
+    )}&fieldToggles=${encodeURIComponent(
+      JSON.stringify({
+        withArticleRichContentState: true
+      })
+    )}`,
+    event,
+    useElongator,
+    (_conversation: unknown) => {
+      const conversation = _conversation as TweetResultsByRestIdResult;
+      // If we get a not found error it's still a valid response
+      const tweet = conversation.data?.tweetResult?.result;
+      if (isGraphQLTweet(tweet)) {
+        return true;
+      }
+      console.log('invalid graphql tweet');
+      if (
+        !tweet &&
+        typeof conversation.data?.tweetResult === 'object' &&
+        Object.keys(conversation.data?.tweetResult || {}).length === 0
+      ) {
+        console.log('tweet was not found');
+        return true;
+      }
+      if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
+        console.log('tweet is nsfw');
+        return true;
+      }
+      if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
+        console.log('tweet is protected');
+        return true;
+      }
+      if (tweet?.__typename === 'TweetUnavailable') {
+        console.log('generic tweet unavailable error');
+        return true;
+      }
+      // Final clause for checking if it's valid is if there's errors
+      return Array.isArray(conversation.errors);
+    }
+  )) as TweetResultsByRestIdResult;
+};
+
+
+const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => {
   const bucket: GraphQLProcessBucket = {
     tweets: [],
     cursors: []
@@ -133,12 +213,7 @@ const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number =>
     console.log('uhhh, we could not even find that tweet, dunno how that happened');
     return -1;
   }
-  const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str);
-  if (index === -1) {
-    console.log('could not find shit for', id)
-    console.log(bucket.cursors)
-  }
-  return index;
+  return bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str);
 }
 
 const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => {
@@ -156,11 +231,39 @@ const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
   return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id)
 }
 
-export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise<SocialThread> => {
-  const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse;
+export const constructTwitterThread = async (id: string,
+  processThread = false,
+  request: IRequest,
+  language: string | undefined,
+  legacyAPI = false): Promise<SocialThread> => {
 
-  if (!response.data) {
-    return { post: null, thread: null, author: null };
+  let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult;
+  let post: APITweet;
+
+  if (typeof TwitterProxy === "undefined" || !experimentCheck(Experiment.TWEET_DETAIL_API)) {
+    console.log('Using TweetResultsByRestId for request...');
+    response = await fetchByRestId(id, request.event) as TweetResultsByRestIdResult;
+
+    const result = response?.data?.tweetResult?.result as GraphQLTweet;
+
+    if (typeof result?.tweet === "undefined") {
+      return { post: null, thread: null, author: null, code: 404 };
+    }
+
+    post = await buildAPITweet(result, language, false, legacyAPI) as APITweet;
+
+    if (post === null) {
+      return { post: null, thread: null, author: null, code: 404 };
+    }
+
+    return { post: post, thread: null, author: post.author, code: 200 };
+  } else {
+    console.log('Using TweetDetail for request...');
+    response = await fetchTweetDetail(id, request.event) as GraphQLTweetFoundResponse;
+
+    if (!response.data) {
+      return { post: null, thread: null, author: null, code: 404 };
+    }
   }
 
   const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions);
@@ -168,23 +271,20 @@ export const processTwitterThread = async (id: string, processThread = false, re
 
   /* Don't bother processing thread on a null tweet */
   if (originalTweet === null) {
-    return { post: null, thread: null, author: null };
+    return { post: null, thread: null, author: null, code: 404 };
   }
-  
-  const post = await buildAPITweet(originalTweet, undefined, false, false);
+    
+  post = await buildAPITweet(originalTweet, undefined, false, legacyAPI) as APITweet;
 
   if (post === null) {
-    return { post: null, thread: null, author: null };
+    return { post: null, thread: null, author: null, code: 404 };
   }
 
   const author = post.author;
-  /* remove post.author */
-  // @ts-expect-error lmao
-  delete post.author;
 
   /* If we're not processing threads, let's be done here */
   if (!processThread) {
-    return { post: post, thread: null, author: author };
+    return { post: post, thread: null, author: author, code: 200 };
   }
 
   const threadTweets = [originalTweet];
@@ -221,7 +321,7 @@ export const processTwitterThread = async (id: string, processThread = false, re
       let loadCursor: GraphQLTweetFoundResponse;
 
       try {
-        loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
+        loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value)
 
         if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
           console.log('Unknown data while fetching cursor', loadCursor);
@@ -267,7 +367,7 @@ export const processTwitterThread = async (id: string, processThread = false, re
       let loadCursor: GraphQLTweetFoundResponse;
 
       try {
-        loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
+        loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value)
 
         if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
           console.log('Unknown data while fetching cursor', loadCursor);
@@ -292,11 +392,12 @@ export const processTwitterThread = async (id: string, processThread = false, re
   const socialThread: SocialThread = {
     post: post,
     thread: [],
-    author: author
+    author: author,
+    code: 200
   }
 
   threadTweets.forEach(async (tweet) => {
-    socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false));
+    socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false) as APITweet);
   });
 
   return socialThread;
@@ -305,7 +406,7 @@ export const processTwitterThread = async (id: string, processThread = false, re
 export const threadAPIProvider = async (request: IRequest) => {
   const { id } = request.params;
 
-  const processedResponse = await processTwitterThread(id, true, request);
+  const processedResponse = await constructTwitterThread(id, true, request, undefined);
 
   return new Response(JSON.stringify(processedResponse), {
     headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }

src/providers/twitter/processor.ts

@@ -74,6 +74,7 @@ export const buildAPITweet = async (
   }
   apiTweet.replies = tweet.legacy.reply_count;
   if (legacyAPI) {
+    // @ts-expect-error Use retweets for legacy API
     apiTweet.retweets = tweet.legacy.retweet_count;
   } else {
     apiTweet.reposts = tweet.legacy.retweet_count;
@@ -206,7 +207,6 @@ export const buildAPITweet = async (
 
   /* If a language is specified in API or by user, let's try translating it! */
   if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
-    /* TODO: Reimplement */
     console.log(`Attempting to translate Tweet to ${language}...`);
     const translateAPI = await translateTweet(tweet, '', language);
     if (translateAPI !== null && translateAPI?.translation) {

src/render/instantview.ts

@@ -4,7 +4,7 @@ import { getSocialTextIV } from '../helpers/author';
 import { sanitizeText } from '../helpers/utils';
 import { Strings } from '../strings';
 
-const populateUserLinks = (tweet: APITweet, text: string): string => {
+const populateUserLinks = (tweet: APIPost, text: string): string => {
   /* TODO: Maybe we can add username splices to our API so only genuinely valid users are linked? */
   text.match(/@(\w{1,15})/g)?.forEach(match => {
     const username = match.replace('@', '');
@@ -117,7 +117,7 @@ const truncateSocialCount = (count: number): string => {
   }
 };
 
-const generateTweetFooter = (tweet: APITweet, isQuote = false): string => {
+const generateTweetFooter = (tweet: APIPost, isQuote = false): string => {
   const { author } = tweet;
 
   let description = author.description;

src/render/photo.ts

@@ -35,6 +35,8 @@ export const renderPhoto = (
     }
   }
 
+  console.log('photo!', photo);
+
   if (photo.type === 'mosaic_photo' && !isOverrideMedia) {
     instructions.addHeaders = [
       `<meta property="twitter:image" content="${tweet.media?.mosaic?.formats.jpeg}"/>`,

src/types/twitterTypes.d.ts

@@ -501,7 +501,7 @@ type GraphQLConversationThread = {
 
 type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown;
 
-type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction;
+type ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction;
 
 type TimelineAddEntriesInstruction = {
   type: 'TimelineAddEntries';
@@ -544,9 +544,10 @@ type GraphQLTweetNotFoundResponse = {
   data: Record<string, never>;
 };
 type GraphQLTweetFoundResponse = {
+  errors?: unknown[];
   data: {
     threaded_conversation_with_injections_v2: {
-      instructions: V2ThreadInstruction[];
+      instructions: ThreadInstruction[];
     };
   };
 };
@@ -555,12 +556,18 @@ type TweetResultsByRestIdResult = {
   errors?: unknown[];
   data?: {
     tweetResult?: {
-      result?:
-        | {
-            __typename: 'TweetUnavailable';
-            reason: 'NsfwLoggedOut' | 'Protected';
-          }
-        | GraphQLTweet;
+      result?: TweetStub | GraphQLTweet;
     };
   };
 };
+
+type TweetStub = {
+  __typename: 'TweetUnavailable';
+  reason: 'NsfwLoggedOut' | 'Protected';
+}
+
+
+interface GraphQLProcessBucket {
+  tweets: GraphQLTweet[];
+  cursors: GraphQLTimelineCursor[];
+}

src/types/types.d.ts

@@ -43,31 +43,18 @@ interface Request {
   };
 }
 
-interface Size {
-  width: number;
-  height: number;
-}
-
-interface HorizontalSize {
-  width: number;
-  height: number;
-  firstWidth: number;
-  secondWidth: number;
-}
-
-interface VerticalSize {
-  width: number;
-  height: number;
-  firstHeight: number;
-  secondHeight: number;
-}
-
 interface TweetAPIResponse {
   code: number;
   message: string;
   tweet?: APITweet;
 }
 
+interface SocialPostAPIResponse {
+  code: number;
+  message: string;
+  post?: APITweet;
+}
+
 interface UserAPIResponse {
   code: number;
   message: string;
@@ -168,7 +155,6 @@ interface APIPost {
 }
 
 interface APITweet extends APIPost {
-  retweets: number;
   views?: number | null;
   translation?: APITranslate;
 
@@ -204,3 +190,15 @@ interface APIUser {
     year?: number;
   };
 }
+
+interface SocialPost {
+  post: APIPost | APITweet | null;
+  author: APIUser | null;
+}
+
+interface SocialThread {
+  post: APIPost | APITweet | null;
+  thread: (APIPost | APITweet)[] | null;
+  author: APIUser | null;
+  code: number;
+}

src/worker.ts

@@ -10,7 +10,7 @@ import { Strings } from './strings';
 import motd from '../motd.json';
 import { sanitizeText } from './helpers/utils';
 import { handleProfile } from './user';
-import { threadAPIProvider } from './providers/twitter/status';
+import { threadAPIProvider } from './providers/twitter/conversation';
 
 declare const globalThis: {
   fetchCompletedTime: number;