From 0a616e973178b57be751d4fbb9867e7b4a2e9207 Mon Sep 17 00:00:00 2001
From: dangered wolf <d@ngeredwolf.me>
Date: Tue, 17 Oct 2023 23:20:46 -0400
Subject: [PATCH] Add twitter thread api (WIP)
---
src/api/status.ts | 20 +-
src/api/user.ts | 8 +-
src/constants.ts | 1 +
src/embed/status.ts | 12 +-
src/providers/twitter/processor.ts | 211 ++++++++++++++++++++
src/providers/twitter/status.ts | 308 +++++++++++++++++++++++++++++
src/types/twitterTypes.d.ts | 69 +++++--
src/types/types.d.ts | 51 ++---
src/worker.ts | 2 +
9 files changed, 627 insertions(+), 55 deletions(-)
create mode 100644 src/providers/twitter/processor.ts
create mode 100644 src/providers/twitter/status.ts
diff --git a/src/api/status.ts b/src/api/status.ts
index 76a6e8c..ef129c7 100644
--- a/src/api/status.ts
+++ b/src/api/status.ts
@@ -56,6 +56,7 @@ const populateTweetProperties = async (
name: apiUser.name,
screen_name: apiUser.screen_name,
avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '',
+ // @ts-expect-error Legacy api shit
avatar_color: null,
banner_url: apiUser.banner_url || '',
description: apiUser.description || '',
@@ -72,8 +73,11 @@ const populateTweetProperties = async (
};
apiTweet.replies = tweet.legacy.reply_count;
apiTweet.retweets = tweet.legacy.retweet_count;
+ apiTweet.reposts = tweet.legacy.retweet_count;
apiTweet.likes = tweet.legacy.favorite_count;
+ // @ts-expect-error Legacy api shit
apiTweet.color = null;
+ // @ts-expect-error legacy api
apiTweet.twitter_card = 'tweet';
apiTweet.created_at = tweet.legacy.created_at;
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
@@ -126,11 +130,11 @@ const populateTweetProperties = async (
apiTweet.media.all.push(mediaObject);
if (mediaObject.type === 'photo') {
- apiTweet.twitter_card = 'summary_large_image';
+ apiTweet.embed_card = 'summary_large_image';
apiTweet.media.photos = apiTweet.media.photos || [];
apiTweet.media.photos.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
- apiTweet.twitter_card = 'player';
+ apiTweet.embed_card = 'player';
apiTweet.media.videos = apiTweet.media.videos || [];
apiTweet.media.videos.push(mediaObject);
} else {
@@ -178,9 +182,9 @@ const populateTweetProperties = async (
/* Workaround: Force player card by default for videos */
/* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
The mediaList however can set it to something else. TODO: Reimplement as enums */
- // @ts-expect-error see above comment
- if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') {
- apiTweet.twitter_card = 'player';
+
+ if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
+ apiTweet.embed_card = 'player';
}
/* If a language is specified in API or by user, let's try translating it! */
@@ -291,9 +295,9 @@ export const statusAPI = async (
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet;
- /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */
- if (apiTweet.twitter_card === 'tweet') {
- apiTweet.twitter_card = apiTweet.quote.twitter_card;
+ /* Only override the embed_card if it's a basic tweet, since media always takes precedence */
+ if (apiTweet.embed_card === 'tweet') {
+ apiTweet.embed_card = apiTweet.quote.embed_card;
}
}
diff --git a/src/api/user.ts b/src/api/user.ts
index 45885eb..499900c 100644
--- a/src/api/user.ts
+++ b/src/api/user.ts
@@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
apiUser.following = user.legacy.friends_count;
apiUser.likes = user.legacy.favourites_count;
apiUser.tweets = user.legacy.statuses_count;
+ apiUser.posts = user.legacy.statuses_count;
apiUser.name = user.legacy.name;
apiUser.screen_name = user.legacy.screen_name;
- apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description);
- apiUser.location = user.legacy.location;
- apiUser.banner_url = user.legacy.profile_banner_url;
+ apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`
+ apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null;
+ apiUser.location = user.legacy.location ? user.legacy.location : null;
+ apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null;
/*
if (user.is_blue_verified) {
apiUser.verified = 'blue';
diff --git a/src/constants.ts b/src/constants.ts
index 5c8ff7e..addbee0 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -15,6 +15,7 @@ export const Constants = {
RELEASE_NAME: RELEASE_NAME,
API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`,
TWITTER_ROOT: 'https://twitter.com',
+ TWITTER_GLOBAL_NAME_ROOT: 'twitter.com',
TWITTER_API_ROOT: 'https://api.twitter.com',
BOT_UA_REGEX:
/bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi,
diff --git a/src/embed/status.ts b/src/embed/status.ts
index 1bab6d8..7f7df01 100644
--- a/src/embed/status.ts
+++ b/src/embed/status.ts
@@ -120,7 +120,7 @@ export const handleStatus = async (
const headers = [
`<link rel="canonical" href="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
`<meta property="og:url" content="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
- `<meta property="theme-color" content="${tweet.color || '#00a8fc'}"/>`,
+ `<meta property="theme-color" content="#00a8fc"/>`,
`<meta property="twitter:site" content="@${tweet.author.screen_name}"/>`,
`<meta property="twitter:creator" content="@${tweet.author.screen_name}"/>`,
`<meta property="twitter:title" content="${tweet.author.name} (@${tweet.author.screen_name})"/>`
@@ -199,8 +199,8 @@ export const handleStatus = async (
siteName = instructions.siteName;
}
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
- if (tweet.twitter_card === 'player') {
- tweet.twitter_card = 'summary_large_image';
+ if (tweet.embed_card === 'player') {
+ tweet.embed_card = 'summary_large_image';
}
break;
case 'video':
@@ -216,8 +216,8 @@ export const handleStatus = async (
siteName = instructions.siteName;
}
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
- if (tweet.twitter_card !== 'player') {
- tweet.twitter_card = 'player';
+ if (tweet.embed_card !== 'player') {
+ tweet.embed_card = 'player';
}
/* This Tweet has a video to render. */
break;
@@ -344,7 +344,7 @@ export const handleStatus = async (
and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */
const text = useIV ? sanitizeText(newText).replace(/\n/g, '<br>') : sanitizeText(newText);
- const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card;
+ const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card;
/* Push basic headers relating to author, Tweet text, and site name */
headers.push(
diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts
new file mode 100644
index 0000000..d28203b
--- /dev/null
+++ b/src/providers/twitter/processor.ts
@@ -0,0 +1,211 @@
+import { renderCard } from '../../helpers/card';
+import { Constants } from '../../constants';
+import { linkFixer } from '../../helpers/linkFixer';
+import { handleMosaic } from '../../helpers/mosaic';
+// import { translateTweet } from '../../helpers/translate';
+import { unescapeText } from '../../helpers/utils';
+import { processMedia } from '../../helpers/media';
+import { convertToApiUser } from '../../api/user';
+
+export const buildAPITweet = async (
+ tweet: GraphQLTweet,
+ language: string | undefined,
+ threadPiece = false,
+ legacyAPI = false
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+): Promise<APITweet> => {
+ const apiTweet = {} as APITweet;
+
+ /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
+ It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */
+
+ if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') {
+ tweet = tweet.result;
+ }
+
+ if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') {
+ tweet.core = tweet.tweet.core;
+ }
+
+ if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') {
+ tweet.legacy = tweet.tweet?.legacy;
+ }
+
+ if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') {
+ tweet.views = tweet?.tweet?.views;
+ }
+
+ const graphQLUser = tweet.core.user_results.result;
+ const apiUser = convertToApiUser(graphQLUser);
+
+ /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */
+ const id = tweet.rest_id ?? tweet.legacy.id_str;
+
+ /* Populating a lot of the basics */
+ apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
+ apiTweet.id = id;
+ apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || ''));
+ if (!threadPiece) {
+ apiTweet.author = {
+ id: apiUser.id,
+ name: apiUser.name,
+ screen_name: apiUser.screen_name,
+ global_screen_name: apiUser.global_screen_name,
+ avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null,
+ banner_url: apiUser.banner_url ?? null,
+ description: apiUser.description ?? null,
+ location: apiUser.location ?? null,
+ url: apiUser.url ?? null,
+ followers: apiUser.followers,
+ following: apiUser.following,
+ joined: apiUser.joined,
+ posts: apiUser.tweets,
+ likes: apiUser.likes,
+ protected: apiUser.protected,
+ birthday: apiUser.birthday,
+ website: apiUser.website
+ };
+ }
+ apiTweet.replies = tweet.legacy.reply_count;
+ if (legacyAPI) {
+ apiTweet.retweets = tweet.legacy.retweet_count;
+ } else {
+ apiTweet.reposts = tweet.legacy.retweet_count;
+ }
+ apiTweet.likes = tweet.legacy.favorite_count;
+ apiTweet.embed_card = 'tweet';
+ apiTweet.created_at = tweet.legacy.created_at;
+ apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
+
+ apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive;
+
+ if (tweet.views.state === 'EnabledWithCount') {
+ apiTweet.views = parseInt(tweet.views.count || '0') ?? null;
+ } else {
+ apiTweet.views = null;
+ }
+ console.log('note_tweet', JSON.stringify(tweet.note_tweet));
+ const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text;
+
+ if (noteTweetText) {
+ tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls;
+ tweet.legacy.entities.hashtags =
+ tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags;
+ tweet.legacy.entities.symbols =
+ tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols;
+
+ console.log('We meet the conditions to use new note tweets');
+ apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText));
+ apiTweet.is_note_tweet = true;
+ } else {
+ apiTweet.is_note_tweet = false;
+ }
+
+ if (tweet.legacy.lang !== 'unk') {
+ apiTweet.lang = tweet.legacy.lang;
+ } else {
+ apiTweet.lang = null;
+ }
+
+ if (legacyAPI) {
+ apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
+ apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
+ } else if (tweet.legacy.in_reply_to_screen_name) {
+ apiTweet.reply_of = {
+ screen_name: tweet.legacy.in_reply_to_screen_name || null,
+ post: tweet.legacy.in_reply_to_status_id_str || null
+ };
+ } else {
+ apiTweet.reply_of = null;
+ }
+
+ apiTweet.media = {
+ all: [],
+ photos: [],
+ videos: [],
+ };
+
+ const mediaList = Array.from(
+ tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []
+ );
+
+ // console.log('tweet', JSON.stringify(tweet));
+
+ /* Populate this Tweet's media */
+ mediaList.forEach(media => {
+ const mediaObject = processMedia(media);
+ if (mediaObject) {
+ apiTweet.media?.all?.push(mediaObject);
+ if (mediaObject.type === 'photo') {
+ apiTweet.embed_card = 'summary_large_image';
+ apiTweet.media?.photos?.push(mediaObject);
+ } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
+ apiTweet.embed_card = 'player';
+ apiTweet.media?.videos?.push(mediaObject);
+ } else {
+ console.log('Unknown media type', mediaObject.type);
+ }
+ }
+ });
+
+ /* Grab color palette data */
+ /*
+ if (mediaList[0]?.ext_media_color?.palette) {
+ apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette);
+ }
+ */
+
+ /* Handle photos and mosaic if available */
+ if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) {
+ const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
+ if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
+ apiTweet.media.mosaic = mosaic;
+ }
+ }
+
+ // Add Tweet source but remove the link HTML tag
+ if (tweet.source) {
+ apiTweet.source = (tweet.source || '').replace(
+ /<a href="(.+?)" rel="nofollow">(.+?)<\/a>/,
+ '$2'
+ );
+ }
+
+ /* Populate a Twitter card */
+
+ if (tweet.card) {
+ const card = renderCard(tweet.card);
+ if (card.external_media) {
+ apiTweet.media = apiTweet.media ?? {};
+ apiTweet.media.external = card.external_media;
+ }
+ if (card.poll) {
+ apiTweet.poll = card.poll;
+ }
+ }
+
+ /* Workaround: Force player card by default for videos */
+ /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
+ The mediaList however can set it to something else. TODO: Reimplement as enums */
+ // @ts-expect-error see above comment
+ if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
+ apiTweet.embed_card = 'player';
+ }
+
+ /* If a language is specified in API or by user, let's try translating it! */
+ if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
+ /* TODO: Reimplement */
+ // console.log(`Attempting to translate Tweet to ${language}...`);
+ // const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language);
+ // if (translateAPI !== null && translateAPI?.translation) {
+ // apiTweet.translation = {
+ // text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')),
+ // source_lang: translateAPI?.sourceLanguage || '',
+ // target_lang: translateAPI?.destinationLanguage || '',
+ // source_lang_en: translateAPI?.localizedSourceLanguage || ''
+ // };
+ // }
+ }
+
+ return apiTweet;
+};
\ No newline at end of file
diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/status.ts
new file mode 100644
index 0000000..f43c141
--- /dev/null
+++ b/src/providers/twitter/status.ts
@@ -0,0 +1,308 @@
+import { IRequest } from "itty-router";
+import { Constants } from "../../constants";
+import { twitterFetch } from "../../fetch";
+import { buildAPITweet } from "./processor";
+
+type GraphQLProcessBucket = {
+ tweets: GraphQLTweet[];
+ cursors: GraphQLTimelineCursor[];
+}
+
+type SocialThread = {
+ post: APIPost | APITweet | null;
+ thread: (APIPost | APITweet)[] | null;
+ author: APIUser | null;
+}
+
+export const fetchTwitterThread = async (
+ status: string,
+ event: FetchEvent,
+ useElongator = typeof TwitterProxy !== 'undefined',
+ cursor: string | null = null
+): Promise<GraphQLTweetFoundResponse> => {
+ return (await twitterFetch(
+ `${
+ Constants.TWITTER_ROOT
+ }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent(
+ JSON.stringify({
+ focalTweetId: status,
+ referrer: "home",
+ with_rux_injections: false,
+ includePromotedContent: false,
+ withCommunity: true,
+ withBirdwatchNotes: true,
+ withQuickPromoteEligibilityTweetFields: false,
+ withVoice: false,
+ withV2Timeline: true,
+ cursor: cursor
+ })
+ )}&features=${encodeURIComponent(
+ JSON.stringify({
+ responsive_web_graphql_exclude_directive_enabled: true,
+ verified_phone_label_enabled: false,
+ responsive_web_home_pinned_timelines_enabled: true,
+ creator_subscriptions_tweet_preview_api_enabled: true,
+ responsive_web_graphql_timeline_navigation_enabled: true,
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+ tweetypie_unmention_optimization_enabled: true,
+ responsive_web_edit_tweet_api_enabled: true,
+ graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+ view_counts_everywhere_api_enabled: true,
+ longform_notetweets_consumption_enabled: true,
+ responsive_web_twitter_article_tweet_consumption_enabled: false,
+ tweet_awards_web_tipping_enabled: false,
+ freedom_of_speech_not_reach_fetch_enabled: true,
+ standardized_nudges_misinfo: true,
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
+ longform_notetweets_rich_text_read_enabled: true,
+ longform_notetweets_inline_media_enabled: true,
+ responsive_web_media_download_video_enabled: true,
+ responsive_web_enhance_cards_enabled: true
+ })
+ )}&fieldToggles=${encodeURIComponent(
+ JSON.stringify({
+ withArticleRichContentState: true
+ })
+ )}`,
+ event,
+ useElongator,
+ () => {
+ return true;
+ }
+ )) as GraphQLTweetFoundResponse;
+};
+
+const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => {
+ const bucket: GraphQLProcessBucket = {
+ tweets: [],
+ cursors: []
+ };
+ instructions.forEach?.(instruction => {
+ if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') {
+ // @ts-expect-error Use entries or moduleItems depending on the type
+ (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => {
+ const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry
+ const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content;
+ if (content.__typename === 'TimelineTimelineItem') {
+ const itemContentType = content.itemContent?.__typename;
+ if (itemContentType === 'TimelineTweet') {
+ const entryType = content.itemContent.tweet_results.result.__typename
+ if (entryType === 'Tweet') {
+ bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet);
+ }
+ if (entryType === 'TweetWithVisibilityResults') {
+ bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet);
+ }
+ } else if (itemContentType === 'TimelineTimelineCursor') {
+ bucket.cursors.push(content.itemContent as GraphQLTimelineCursor);
+ }
+ } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') {
+ content.items.forEach((item) => {
+ const itemContentType = item.item.itemContent.__typename;
+ if (itemContentType === 'TimelineTweet') {
+ const entryType = item.item.itemContent.tweet_results.result.__typename
+ if (entryType === 'Tweet') {
+ bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet);
+ }
+ if (entryType === 'TweetWithVisibilityResults') {
+ bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet);
+ }
+ } else if (itemContentType === 'TimelineTimelineCursor') {
+ bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor);
+ }
+ });
+ }
+ });
+ }
+ })
+
+ return bucket;
+}
+
+const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => {
+ return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null;
+}
+
+const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => {
+ return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id);
+}
+
+const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => {
+ const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id);
+ if (!tweet) {
+ console.log('uhhh, we could not even find that tweet, dunno how that happened');
+ return -1;
+ }
+ const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str);
+ if (index === -1) {
+ console.log('could not find shit for', id)
+ console.log(bucket.cursors)
+ }
+ return index;
+}
+
+const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => {
+ /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */
+ return oldCursors.map(cursor => {
+ const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType);
+ if (newCursor) {
+ return newCursor;
+ }
+ return cursor;
+ });
+}
+
+const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
+ return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id)
+}
+
+export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise<SocialThread> => {
+ const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse;
+
+ if (!response.data) {
+ return { post: null, thread: null, author: null };
+ }
+
+ const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions);
+ const originalTweet = findTweetInBucket(id, bucket);
+
+ /* Don't bother processing thread on a null tweet */
+ if (originalTweet === null) {
+ return { post: null, thread: null, author: null };
+ }
+
+ const post = await buildAPITweet(originalTweet, undefined, false, false);
+ const author = post.author;
+ /* remove post.author */
+ // @ts-expect-error lmao
+ delete post.author;
+
+ /* If we're not processing threads, let's be done here */
+ if (!processThread) {
+ return { post: post, thread: null, author: author };
+ }
+
+ const threadTweets = [originalTweet];
+ bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet);
+
+ let currentId = id;
+
+ /* Process tweets that are following the current one in the thread */
+ while (findNextTweet(currentId, bucket) !== -1) {
+ const index = findNextTweet(currentId, bucket);
+ const tweet = bucket.tweets[index];
+
+ const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
+
+ console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
+
+ threadTweets.push(tweet);
+
+ currentId = newCurrentId;
+
+ console.log('Current index', index, 'of', bucket.tweets.length)
+
+ /* Reached the end of the current list of tweets in thread) */
+ if (index >= (bucket.tweets.length - 1)) {
+ /* See if we have a cursor to fetch more tweets */
+ const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore'));
+ console.log('current cursors: ', bucket.cursors)
+ if (!cursor) {
+ console.log('No cursor present, stopping pagination down')
+ break;
+ }
+ console.log('Cursor present, fetching more tweets down');
+
+ let loadCursor: GraphQLTweetFoundResponse;
+
+ try {
+ loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
+
+ if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
+ console.log('Unknown data while fetching cursor', loadCursor);
+ break;
+ }
+ } catch(e) {
+ console.log('Error fetching cursor', e);
+ break;
+ }
+
+ const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
+ bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet));
+ /* Remove old cursor and add new bottom cursor if necessary */
+ consolidateCursors(bucket.cursors, cursorResponse.cursors);
+ console.log('updated bucket of cursors', bucket.cursors);
+ }
+
+ console.log('Preview of next tweet:', findNextTweet(currentId, bucket));
+ }
+
+ currentId = id;
+
+ while (findPreviousTweet(currentId, bucket) !== -1) {
+ const index = findPreviousTweet(currentId, bucket);
+ const tweet = bucket.tweets[index];
+ const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
+
+ console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
+
+ threadTweets.unshift(tweet);
+
+ currentId = newCurrentId;
+
+ if (index === 0) {
+ /* See if we have a cursor to fetch more tweets */
+ const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore'));
+ console.log('current cursors: ', bucket.cursors)
+ if (!cursor) {
+ console.log('No cursor present, stopping pagination up')
+ break;
+ }
+ console.log('Cursor present, fetching more tweets up');
+ let loadCursor: GraphQLTweetFoundResponse;
+
+ try {
+ loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
+
+ if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
+ console.log('Unknown data while fetching cursor', loadCursor);
+ break;
+ }
+ } catch(e) {
+ console.log('Error fetching cursor', e);
+ break;
+ }
+ const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
+ bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet));
+ /* Remove old cursor and add new top cursor if necessary */
+ consolidateCursors(bucket.cursors, cursorResponse.cursors);
+
+ // console.log('updated bucket of tweets', bucket.tweets);
+ console.log('updated bucket of cursors', bucket.cursors);
+ }
+
+ console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket));
+ }
+
+ const socialThread: SocialThread = {
+ post: post,
+ thread: [],
+ author: author
+ }
+
+ threadTweets.forEach(async (tweet) => {
+ socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false));
+ });
+
+ return socialThread;
+}
+
+export const threadAPIProvider = async (request: IRequest) => {
+ const { id } = request.params;
+
+ const processedResponse = await processTwitterThread(id, true, request);
+
+ return new Response(JSON.stringify(processedResponse), {
+ headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }
+ })
+}
\ No newline at end of file
diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts
index 35bce9b..8757117 100644
--- a/src/types/twitterTypes.d.ts
+++ b/src/types/twitterTypes.d.ts
@@ -354,7 +354,7 @@ type GraphQLTweetLegacy = {
type GraphQLTweet = {
// Workaround
result: GraphQLTweet;
- __typename: 'Tweet' | 'TweetUnavailable';
+ __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable';
rest_id: string; // "1674824189176590336",
has_birdwatch_notes: false;
core: {
@@ -444,37 +444,76 @@ type TweetTombstone = {
};
};
};
+
+type GraphQLTimelineTweet = {
+ item: 'TimelineTweet';
+ __typename: 'TimelineTweet';
+ tweet_results: {
+ result: GraphQLTweet | TweetTombstone;
+ };
+}
+
+type GraphQLTimelineCursor = {
+ cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore';
+ itemType: 'TimelineTimelineCursor';
+ value: string;
+ __typename: 'TimelineTimelineCursor';
+}
+
+interface GraphQLBaseTimeline {
+ entryType: string;
+ __typename: string;
+}
+
+type GraphQLTimelineItem = GraphQLBaseTimeline & {
+ entryType: 'TimelineTimelineItem';
+ __typename: 'TimelineTimelineItem';
+ itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor;
+}
+
+type GraphQLTimelineModule = GraphQLBaseTimeline & {
+ entryType: 'TimelineTimelineModule';
+ __typename: 'TimelineTimelineModule';
+ items: {
+ entryId: `conversationthread-${number}-tweet-${number}`;
+ item: GraphQLTimelineItem
+ }[];
+}
+
type GraphQLTimelineTweetEntry = {
/** The entryID contains the tweet ID */
entryId: `tweet-${number}`; // "tweet-1674824189176590336"
sortIndex: string;
- content: {
- entryType: 'TimelineTimelineItem';
- __typename: 'TimelineTimelineItem';
- itemContent: {
- item: 'TimelineTweet';
- __typename: 'TimelineTweet';
- tweet_results: {
- result: GraphQLTweet | TweetTombstone;
- };
- };
- };
+ content: GraphQLTimelineItem;
};
+
+type GraphQLModuleTweetEntry = {
+ /** The entryID contains the tweet ID */
+ sortIndex: string;
+ item: GraphQLTimelineItem | GraphQLTimelineModule;
+};
+
type GraphQLConversationThread = {
entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336"
sortIndex: string;
+ content: GraphQLTimelineModule;
};
type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown;
-type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction;
+type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction;
-type TimeLineAddEntriesInstruction = {
+type TimelineAddEntriesInstruction = {
type: 'TimelineAddEntries';
entries: GraphQLTimelineEntry[];
};
-type TimeLineTerminateTimelineInstruction = {
+type TimelineAddModulesInstruction = {
+ type: 'TimelineAddToModule';
+ moduleItems: GraphQLTimelineEntry[];
+};
+
+type TimelineTerminateTimelineInstruction = {
type: 'TimelineTerminateTimeline';
direction: 'Top';
};
diff --git a/src/types/types.d.ts b/src/types/types.d.ts
index d6f9f7f..e911d81 100644
--- a/src/types/types.d.ts
+++ b/src/types/types.d.ts
@@ -81,14 +81,6 @@ interface APITranslate {
target_lang: string;
}
-interface BaseUser {
- id?: string;
- name?: string;
- screen_name?: string;
- avatar_url?: string;
- banner_url?: string;
-}
-
interface APIExternalMedia {
type: 'video';
url: string;
@@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia {
};
}
-interface APITweet {
+interface APIPost {
id: string;
url: string;
text: string;
@@ -144,15 +136,11 @@ interface APITweet {
created_timestamp: number;
likes: number;
- retweets: number;
+ reposts: number;
replies: number;
- views?: number | null;
- color: string | null;
-
- quote?: APITweet;
+ quote?: APIPost;
poll?: APIPoll;
- translation?: APITranslate;
author: APIUser;
media?: {
@@ -169,24 +157,41 @@ interface APITweet {
replying_to: string | null;
replying_to_status: string | null;
- source: string;
+ reply_of: {
+ screen_name: string | null;
+ post: string | null;
+ } | null
- is_note_tweet: boolean;
+ source: string | null;
- twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
+ embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
}
-interface APIUser extends BaseUser {
+interface APITweet extends APIPost {
+ retweets: number;
+ views?: number | null;
+ translation?: APITranslate;
+
+ is_note_tweet: boolean;
+}
+
+interface APIUser {
+ id: string;
+ name: string;
+ screen_name: string;
+ global_screen_name: string;
+ avatar_url: string | null;
+ banner_url: string | null;
// verified: 'legacy' | 'blue'| 'business' | 'government';
// verified_label: string;
- description: string;
- location: string;
+ description: string | null;
+ location: string | null;
url: string;
- avatar_color?: string | null;
protected: boolean;
followers: number;
following: number;
- tweets: number;
+ tweets?: number;
+ posts?: number;
likes: number;
joined: string;
website: {
diff --git a/src/worker.ts b/src/worker.ts
index f3435f9..83f3d18 100644
--- a/src/worker.ts
+++ b/src/worker.ts
@@ -10,6 +10,7 @@ import { Strings } from './strings';
import motd from '../motd.json';
import { sanitizeText } from './helpers/utils';
import { handleProfile } from './user';
+import { threadAPIProvider } from './providers/twitter/status';
declare const globalThis: {
fetchCompletedTime: number;
@@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest);
router.get('/status/:id/:language', statusRequest);
router.get('/version', versionRequest);
router.get('/set_base_redirect', setRedirectRequest);
+router.get('/v2/twitter/thread/:id', threadAPIProvider)
/* Oembeds (used by Discord to enhance responses)