diff --git a/src/api/status.ts b/src/api/status.ts index 76a6e8c..ef129c7 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -56,6 +56,7 @@ const populateTweetProperties = async ( name: apiUser.name, screen_name: apiUser.screen_name, avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', + // @ts-expect-error Legacy api shit avatar_color: null, banner_url: apiUser.banner_url || '', description: apiUser.description || '', @@ -72,8 +73,11 @@ const populateTweetProperties = async ( }; apiTweet.replies = tweet.legacy.reply_count; apiTweet.retweets = tweet.legacy.retweet_count; + apiTweet.reposts = tweet.legacy.retweet_count; apiTweet.likes = tweet.legacy.favorite_count; + // @ts-expect-error Legacy api shit apiTweet.color = null; + // @ts-expect-error legacy api apiTweet.twitter_card = 'tweet'; apiTweet.created_at = tweet.legacy.created_at; apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; @@ -126,11 +130,11 @@ const populateTweetProperties = async ( apiTweet.media.all.push(mediaObject); if (mediaObject.type === 'photo') { - apiTweet.twitter_card = 'summary_large_image'; + apiTweet.embed_card = 'summary_large_image'; apiTweet.media.photos = apiTweet.media.photos || []; apiTweet.media.photos.push(mediaObject); } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { - apiTweet.twitter_card = 'player'; + apiTweet.embed_card = 'player'; apiTweet.media.videos = apiTweet.media.videos || []; apiTweet.media.videos.push(mediaObject); } else { @@ -178,9 +182,9 @@ const populateTweetProperties = async ( /* Workaround: Force player card by default for videos */ /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' The mediaList however can set it to something else. TODO: Reimplement as enums */ - // @ts-expect-error see above comment - if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') { - apiTweet.twitter_card = 'player'; + + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; } /* If a language is specified in API or by user, let's try translating it! */ @@ -291,9 +295,9 @@ export const statusAPI = async ( const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet; - /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.twitter_card === 'tweet') { - apiTweet.twitter_card = apiTweet.quote.twitter_card; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet') { + apiTweet.embed_card = apiTweet.quote.embed_card; } } diff --git a/src/api/user.ts b/src/api/user.ts index 45885eb..499900c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.following = user.legacy.friends_count; apiUser.likes = user.legacy.favourites_count; apiUser.tweets = user.legacy.statuses_count; + apiUser.posts = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description); - apiUser.location = user.legacy.location; - apiUser.banner_url = user.legacy.profile_banner_url; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` + apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null; + apiUser.location = user.legacy.location ? user.legacy.location : null; + apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/constants.ts b/src/constants.ts index 5c8ff7e..addbee0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,7 @@ export const Constants = { RELEASE_NAME: RELEASE_NAME, API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`, TWITTER_ROOT: 'https://twitter.com', + TWITTER_GLOBAL_NAME_ROOT: 'twitter.com', TWITTER_API_ROOT: 'https://api.twitter.com', BOT_UA_REGEX: /bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi, diff --git a/src/embed/status.ts b/src/embed/status.ts index 1bab6d8..7f7df01 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -120,7 +120,7 @@ export const handleStatus = async ( const headers = [ ``, ``, - ``, + ``, ``, ``, `` @@ -199,8 +199,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card === 'player') { - tweet.twitter_card = 'summary_large_image'; + if (tweet.embed_card === 'player') { + tweet.embed_card = 'summary_large_image'; } break; case 'video': @@ -216,8 +216,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card !== 'player') { - tweet.twitter_card = 'player'; + if (tweet.embed_card !== 'player') { + tweet.embed_card = 'player'; } /* This Tweet has a video to render. */ break; @@ -344,7 +344,7 @@ export const handleStatus = async ( and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */ const text = useIV ? sanitizeText(newText).replace(/\n/g, '
') : sanitizeText(newText); - const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card; + const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card; /* Push basic headers relating to author, Tweet text, and site name */ headers.push( diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts new file mode 100644 index 0000000..d28203b --- /dev/null +++ b/src/providers/twitter/processor.ts @@ -0,0 +1,211 @@ +import { renderCard } from '../../helpers/card'; +import { Constants } from '../../constants'; +import { linkFixer } from '../../helpers/linkFixer'; +import { handleMosaic } from '../../helpers/mosaic'; +// import { translateTweet } from '../../helpers/translate'; +import { unescapeText } from '../../helpers/utils'; +import { processMedia } from '../../helpers/media'; +import { convertToApiUser } from '../../api/user'; + +export const buildAPITweet = async ( + tweet: GraphQLTweet, + language: string | undefined, + threadPiece = false, + legacyAPI = false + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const apiTweet = {} as APITweet; + + /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. + It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ + + if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { + tweet = tweet.result; + } + + if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { + tweet.core = tweet.tweet.core; + } + + if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { + tweet.legacy = tweet.tweet?.legacy; + } + + if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { + tweet.views = tweet?.tweet?.views; + } + + const graphQLUser = tweet.core.user_results.result; + const apiUser = convertToApiUser(graphQLUser); + + /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ + const id = tweet.rest_id ?? tweet.legacy.id_str; + + /* Populating a lot of the basics */ + apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; + apiTweet.id = id; + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); + if (!threadPiece) { + apiTweet.author = { + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, + global_screen_name: apiUser.global_screen_name, + avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, + banner_url: apiUser.banner_url ?? null, + description: apiUser.description ?? null, + location: apiUser.location ?? null, + url: apiUser.url ?? null, + followers: apiUser.followers, + following: apiUser.following, + joined: apiUser.joined, + posts: apiUser.tweets, + likes: apiUser.likes, + protected: apiUser.protected, + birthday: apiUser.birthday, + website: apiUser.website + }; + } + apiTweet.replies = tweet.legacy.reply_count; + if (legacyAPI) { + apiTweet.retweets = tweet.legacy.retweet_count; + } else { + apiTweet.reposts = tweet.legacy.retweet_count; + } + apiTweet.likes = tweet.legacy.favorite_count; + apiTweet.embed_card = 'tweet'; + apiTweet.created_at = tweet.legacy.created_at; + apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; + + apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; + + if (tweet.views.state === 'EnabledWithCount') { + apiTweet.views = parseInt(tweet.views.count || '0') ?? null; + } else { + apiTweet.views = null; + } + console.log('note_tweet', JSON.stringify(tweet.note_tweet)); + const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; + + if (noteTweetText) { + tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; + tweet.legacy.entities.hashtags = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; + tweet.legacy.entities.symbols = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; + + console.log('We meet the conditions to use new note tweets'); + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); + apiTweet.is_note_tweet = true; + } else { + apiTweet.is_note_tweet = false; + } + + if (tweet.legacy.lang !== 'unk') { + apiTweet.lang = tweet.legacy.lang; + } else { + apiTweet.lang = null; + } + + if (legacyAPI) { + apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; + } else if (tweet.legacy.in_reply_to_screen_name) { + apiTweet.reply_of = { + screen_name: tweet.legacy.in_reply_to_screen_name || null, + post: tweet.legacy.in_reply_to_status_id_str || null + }; + } else { + apiTweet.reply_of = null; + } + + apiTweet.media = { + all: [], + photos: [], + videos: [], + }; + + const mediaList = Array.from( + tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] + ); + + // console.log('tweet', JSON.stringify(tweet)); + + /* Populate this Tweet's media */ + mediaList.forEach(media => { + const mediaObject = processMedia(media); + if (mediaObject) { + apiTweet.media?.all?.push(mediaObject); + if (mediaObject.type === 'photo') { + apiTweet.embed_card = 'summary_large_image'; + apiTweet.media?.photos?.push(mediaObject); + } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { + apiTweet.embed_card = 'player'; + apiTweet.media?.videos?.push(mediaObject); + } else { + console.log('Unknown media type', mediaObject.type); + } + } + }); + + /* Grab color palette data */ + /* + if (mediaList[0]?.ext_media_color?.palette) { + apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); + } + */ + + /* Handle photos and mosaic if available */ + if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) { + const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); + if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { + apiTweet.media.mosaic = mosaic; + } + } + + // Add Tweet source but remove the link HTML tag + if (tweet.source) { + apiTweet.source = (tweet.source || '').replace( + /(.+?)<\/a>/, + '$2' + ); + } + + /* Populate a Twitter card */ + + if (tweet.card) { + const card = renderCard(tweet.card); + if (card.external_media) { + apiTweet.media = apiTweet.media ?? {}; + apiTweet.media.external = card.external_media; + } + if (card.poll) { + apiTweet.poll = card.poll; + } + } + + /* Workaround: Force player card by default for videos */ + /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' + The mediaList however can set it to something else. TODO: Reimplement as enums */ + // @ts-expect-error see above comment + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; + } + + /* If a language is specified in API or by user, let's try translating it! */ + if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { + /* TODO: Reimplement */ + // console.log(`Attempting to translate Tweet to ${language}...`); + // const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); + // if (translateAPI !== null && translateAPI?.translation) { + // apiTweet.translation = { + // text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + // source_lang: translateAPI?.sourceLanguage || '', + // target_lang: translateAPI?.destinationLanguage || '', + // source_lang_en: translateAPI?.localizedSourceLanguage || '' + // }; + // } + } + + return apiTweet; +}; \ No newline at end of file diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/status.ts new file mode 100644 index 0000000..f43c141 --- /dev/null +++ b/src/providers/twitter/status.ts @@ -0,0 +1,308 @@ +import { IRequest } from "itty-router"; +import { Constants } from "../../constants"; +import { twitterFetch } from "../../fetch"; +import { buildAPITweet } from "./processor"; + +type GraphQLProcessBucket = { + tweets: GraphQLTweet[]; + cursors: GraphQLTimelineCursor[]; +} + +type SocialThread = { + post: APIPost | APITweet | null; + thread: (APIPost | APITweet)[] | null; + author: APIUser | null; +} + +export const fetchTwitterThread = async ( + status: string, + event: FetchEvent, + useElongator = typeof TwitterProxy !== 'undefined', + cursor: string | null = null +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( + JSON.stringify({ + focalTweetId: status, + referrer: "home", + with_rux_injections: false, + includePromotedContent: false, + withCommunity: true, + withBirdwatchNotes: true, + withQuickPromoteEligibilityTweetFields: false, + withVoice: false, + withV2Timeline: true, + cursor: cursor + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_home_pinned_timelines_enabled: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_media_download_video_enabled: true, + responsive_web_enhance_cards_enabled: true + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + () => { + return true; + } + )) as GraphQLTweetFoundResponse; +}; + +const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => { + const bucket: GraphQLProcessBucket = { + tweets: [], + cursors: [] + }; + instructions.forEach?.(instruction => { + if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { + // @ts-expect-error Use entries or moduleItems depending on the type + (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => { + const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry + const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + if (content.__typename === 'TimelineTimelineItem') { + const itemContentType = content.itemContent?.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = content.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); + } + } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') { + content.items.forEach((item) => { + const itemContentType = item.item.itemContent.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = item.item.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); + } + }); + } + }); + } + }) + + return bucket; +} + +const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { + return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; +} + +const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { + return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); +} + +const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { + const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); + if (!tweet) { + console.log('uhhh, we could not even find that tweet, dunno how that happened'); + return -1; + } + const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); + if (index === -1) { + console.log('could not find shit for', id) + console.log(bucket.cursors) + } + return index; +} + +const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { + /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ + return oldCursors.map(cursor => { + const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); + if (newCursor) { + return newCursor; + } + return cursor; + }); +} + +const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { + return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) +} + +export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise => { + const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse; + + if (!response.data) { + return { post: null, thread: null, author: null }; + } + + const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); + const originalTweet = findTweetInBucket(id, bucket); + + /* Don't bother processing thread on a null tweet */ + if (originalTweet === null) { + return { post: null, thread: null, author: null }; + } + + const post = await buildAPITweet(originalTweet, undefined, false, false); + const author = post.author; + /* remove post.author */ + // @ts-expect-error lmao + delete post.author; + + /* If we're not processing threads, let's be done here */ + if (!processThread) { + return { post: post, thread: null, author: author }; + } + + const threadTweets = [originalTweet]; + bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet); + + let currentId = id; + + /* Process tweets that are following the current one in the thread */ + while (findNextTweet(currentId, bucket) !== -1) { + const index = findNextTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.push(tweet); + + currentId = newCurrentId; + + console.log('Current index', index, 'of', bucket.tweets.length) + + /* Reached the end of the current list of tweets in thread) */ + if (index >= (bucket.tweets.length - 1)) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination down') + break; + } + console.log('Cursor present, fetching more tweets down'); + + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet)); + /* Remove old cursor and add new bottom cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of next tweet:', findNextTweet(currentId, bucket)); + } + + currentId = id; + + while (findPreviousTweet(currentId, bucket) !== -1) { + const index = findPreviousTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.unshift(tweet); + + currentId = newCurrentId; + + if (index === 0) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination up') + break; + } + console.log('Cursor present, fetching more tweets up'); + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet)); + /* Remove old cursor and add new top cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + + // console.log('updated bucket of tweets', bucket.tweets); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket)); + } + + const socialThread: SocialThread = { + post: post, + thread: [], + author: author + } + + threadTweets.forEach(async (tweet) => { + socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false)); + }); + + return socialThread; +} + +export const threadAPIProvider = async (request: IRequest) => { + const { id } = request.params; + + const processedResponse = await processTwitterThread(id, true, request); + + return new Response(JSON.stringify(processedResponse), { + headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } + }) +} \ No newline at end of file diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 35bce9b..8757117 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -354,7 +354,7 @@ type GraphQLTweetLegacy = { type GraphQLTweet = { // Workaround result: GraphQLTweet; - __typename: 'Tweet' | 'TweetUnavailable'; + __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -444,37 +444,76 @@ type TweetTombstone = { }; }; }; + +type GraphQLTimelineTweet = { + item: 'TimelineTweet'; + __typename: 'TimelineTweet'; + tweet_results: { + result: GraphQLTweet | TweetTombstone; + }; +} + +type GraphQLTimelineCursor = { + cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; + itemType: 'TimelineTimelineCursor'; + value: string; + __typename: 'TimelineTimelineCursor'; +} + +interface GraphQLBaseTimeline { + entryType: string; + __typename: string; +} + +type GraphQLTimelineItem = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineItem'; + __typename: 'TimelineTimelineItem'; + itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; +} + +type GraphQLTimelineModule = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineModule'; + __typename: 'TimelineTimelineModule'; + items: { + entryId: `conversationthread-${number}-tweet-${number}`; + item: GraphQLTimelineItem + }[]; +} + type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ entryId: `tweet-${number}`; // "tweet-1674824189176590336" sortIndex: string; - content: { - entryType: 'TimelineTimelineItem'; - __typename: 'TimelineTimelineItem'; - itemContent: { - item: 'TimelineTweet'; - __typename: 'TimelineTweet'; - tweet_results: { - result: GraphQLTweet | TweetTombstone; - }; - }; - }; + content: GraphQLTimelineItem; }; + +type GraphQLModuleTweetEntry = { + /** The entryID contains the tweet ID */ + sortIndex: string; + item: GraphQLTimelineItem | GraphQLTimelineModule; +}; + type GraphQLConversationThread = { entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336" sortIndex: string; + content: GraphQLTimelineModule; }; type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction; +type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; -type TimeLineAddEntriesInstruction = { +type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; entries: GraphQLTimelineEntry[]; }; -type TimeLineTerminateTimelineInstruction = { +type TimelineAddModulesInstruction = { + type: 'TimelineAddToModule'; + moduleItems: GraphQLTimelineEntry[]; +}; + +type TimelineTerminateTimelineInstruction = { type: 'TimelineTerminateTimeline'; direction: 'Top'; }; diff --git a/src/types/types.d.ts b/src/types/types.d.ts index d6f9f7f..e911d81 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -81,14 +81,6 @@ interface APITranslate { target_lang: string; } -interface BaseUser { - id?: string; - name?: string; - screen_name?: string; - avatar_url?: string; - banner_url?: string; -} - interface APIExternalMedia { type: 'video'; url: string; @@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia { }; } -interface APITweet { +interface APIPost { id: string; url: string; text: string; @@ -144,15 +136,11 @@ interface APITweet { created_timestamp: number; likes: number; - retweets: number; + reposts: number; replies: number; - views?: number | null; - color: string | null; - - quote?: APITweet; + quote?: APIPost; poll?: APIPoll; - translation?: APITranslate; author: APIUser; media?: { @@ -169,24 +157,41 @@ interface APITweet { replying_to: string | null; replying_to_status: string | null; - source: string; + reply_of: { + screen_name: string | null; + post: string | null; + } | null - is_note_tweet: boolean; + source: string | null; - twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; + embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; } -interface APIUser extends BaseUser { +interface APITweet extends APIPost { + retweets: number; + views?: number | null; + translation?: APITranslate; + + is_note_tweet: boolean; +} + +interface APIUser { + id: string; + name: string; + screen_name: string; + global_screen_name: string; + avatar_url: string | null; + banner_url: string | null; // verified: 'legacy' | 'blue'| 'business' | 'government'; // verified_label: string; - description: string; - location: string; + description: string | null; + location: string | null; url: string; - avatar_color?: string | null; protected: boolean; followers: number; following: number; - tweets: number; + tweets?: number; + posts?: number; likes: number; joined: string; website: { diff --git a/src/worker.ts b/src/worker.ts index f3435f9..83f3d18 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,6 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; +import { threadAPIProvider } from './providers/twitter/status'; declare const globalThis: { fetchCompletedTime: number; @@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); +router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses)