diff --git a/.env.example b/.env.example index 783c759..c37f8d5 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ STANDARD_DOMAIN_LIST = "fxtwitter.com,fixupx.com,twittpr.com" DIRECT_MEDIA_DOMAINS = "d.fxtwitter.com,dl.fxtwitter.com,d.twittpr.com,dl.twittpr.com,d.fixupx.com,d.xfixup.com,dl.fixupx.com,dl.xfixup.com" TEXT_ONLY_DOMAINS = "t.fxtwitter.com,t.twittpr.com,t.fixupx.com" INSTANT_VIEW_DOMAINS = "i.fxtwitter.com,i.twittpr.com,i.fixupx.com" +INSTANT_VIEW_THREADS_DOMAINS = "u.fxtwitter.com,u.twittpr.com,u.fixupx.com" GALLERY_DOMAINS = "g.fxtwitter.com,g.twittpr.com,g.fixupx.com" NATIVE_MULTI_IMAGE_DOMAINS = "m.fxtwitter.com,m.twittpr.com,m.fixupx.com" MOSAIC_DOMAIN_LIST = "mosaic.fxtwitter.com" diff --git a/esbuild.config.mjs b/esbuild.config.mjs index b6596d6..12b749e 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -37,6 +37,7 @@ let envVariables = [ 'DIRECT_MEDIA_DOMAINS', 'TEXT_ONLY_DOMAINS', 'INSTANT_VIEW_DOMAINS', + 'INSTANT_VIEW_THREADS_DOMAINS', 'GALLERY_DOMAINS', 'NATIVE_MULTI_IMAGE_DOMAINS', 'HOST_URL', @@ -56,14 +57,10 @@ for (let envVar of envVariables) { defines['RELEASE_NAME'] = `"${releaseName}"`; -await esbuild.build({ - entryPoints: ['src/worker.ts'], - sourcemap: 'external', - outdir: 'dist', - minify: true, - bundle: true, - format: 'esm', - plugins: [ +const plugins = []; + +if (process.env.SENTRY_DSN) { + plugins.push( sentryEsbuildPlugin({ org: process.env.SENTRY_ORG, project: process.env.SENTRY_PROJECT, @@ -84,7 +81,16 @@ await esbuild.build({ // https://sentry.io/orgredirect/organizations/:orgslug/settings/auth-tokens/ authToken: process.env.SENTRY_AUTH_TOKEN }) - ], + ); +} +await esbuild.build({ + entryPoints: ['src/worker.ts'], + sourcemap: 'external', + outdir: 'dist', + minify: true, + bundle: true, + format: 'esm', + plugins: plugins, define: defines }); diff --git a/jestconfig.json b/jestconfig.json index 3e44124..44398d8 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -7,6 +7,7 @@ "BRANDING_NAME": "FxTwitter / FixupX", "TEXT_ONLY_DOMAINS": "t.fxtwitter.com,t.twittpr.com,t.fixupx.com", "INSTANT_VIEW_DOMAINS": "i.fxtwitter.com,i.twittpr.com,i.fixupx.com", + "INSTANT_VIEW_THREADS_DOMAINS": "u.fxtwitter.com,u.twittpr.com,u.fixupx.com", "GALLERY_DOMAINS": "g.fxtwitter.com,g.twittpr.com,g.fixupx.com", "NATIVE_MULTI_IMAGE_DOMAINS": "m.fxtwitter.com,m.twittpr.com,m.fixupx.com", "STANDARD_DOMAIN_LIST": "fxtwitter.com,fixupx.com,twittpr.com", @@ -17,6 +18,7 @@ "REDIRECT_URL": "https://github.com/FixTweet/FxTwitter", "EMBED_URL": "https://github.com/FixTweet/FxTwitter", "RELEASE_NAME": "fixtweet-test", + "GIF_TRANSCODE_DOMAIN": "gif.fxtwitter.com", "SENTRY_DSN": null }, "testRegex": "/test/.*\\.test\\.ts$", diff --git a/src/constants.ts b/src/constants.ts index 21dd34b..2426cab 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -5,6 +5,7 @@ export const Constants = { DIRECT_MEDIA_DOMAINS: DIRECT_MEDIA_DOMAINS.split(','), TEXT_ONLY_DOMAINS: TEXT_ONLY_DOMAINS.split(','), INSTANT_VIEW_DOMAINS: INSTANT_VIEW_DOMAINS.split(','), + INSTANT_VIEW_THREADS_DOMAINS: INSTANT_VIEW_THREADS_DOMAINS.split(','), GALLERY_DOMAINS: GALLERY_DOMAINS.split(','), NATIVE_MULTI_IMAGE_DOMAINS: NATIVE_MULTI_IMAGE_DOMAINS.split(','), MOSAIC_DOMAIN_LIST: MOSAIC_DOMAIN_LIST.split(','), diff --git a/src/embed/status.ts b/src/embed/status.ts index 520d772..4205a92 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -36,9 +36,12 @@ export const handleStatus = async ( let fetchWithThreads = false; - /* TODO: Enable actually pulling threads once we can actually do something with them */ - if (c.req.header('user-agent')?.includes('Telegram') && !flags?.direct) { - fetchWithThreads = false; + if ( + c.req.header('user-agent')?.includes('Telegram') && + !flags?.direct && + flags.instantViewUnrollThreads + ) { + fetchWithThreads = true; } const thread = await constructTwitterThread( @@ -111,6 +114,7 @@ export const handleStatus = async ( status.is_note_tweet || status.quote || status.translation || + status.community_note || flags?.forceInstantView); /* Force enable IV for archivers */ @@ -193,6 +197,7 @@ export const handleStatus = async ( try { const instructions = renderInstantView({ status: status, + thread: thread, text: newText, flags: flags }); diff --git a/src/experiments.ts b/src/experiments.ts index 1e9fee4..0b16983 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -1,4 +1,5 @@ export enum Experiment { + IV_FORCE_THREAD_UNROLL = 'IV_FORCE_THREAD_UNROLL', ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT', ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API', TWEET_DETAIL_API = 'TWEET_DETAIL_API', @@ -13,6 +14,11 @@ type ExperimentConfig = { }; const Experiments: { [key in Experiment]: ExperimentConfig } = { + [Experiment.IV_FORCE_THREAD_UNROLL]: { + name: 'IV force thread unroll', + description: 'Force thread unroll for IVs', + percentage: 0.1 + }, [Experiment.ELONGATOR_BY_DEFAULT]: { name: 'Elongator by default', description: 'Enable Elongator by default (guest token lockout bypass)', diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index bd61123..c61cc60 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -90,14 +90,16 @@ export const fetchTweetDetail = async ( useElongator, (_conversation: unknown) => { const conversation = _conversation as TweetDetailResult; - const tweet = findStatusInBucket( - status, - processResponse(conversation?.data?.threaded_conversation_with_injections_v2?.instructions) + const response = processResponse( + conversation?.data?.threaded_conversation_with_injections_v2?.instructions ); + const tweet = findStatusInBucket(status, response); if (tweet && isGraphQLTwitterStatus(tweet)) { return true; } - console.log('invalid graphql tweet', conversation); + console.log('invalid graphql tweet', tweet); + console.log('finding status', status); + console.log('from response', JSON.stringify(response)); return Array.isArray(conversation?.errors); }, @@ -190,6 +192,7 @@ export const fetchByRestId = async ( const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => { const bucket: GraphQLProcessBucket = { statuses: [], + allStatuses: [], cursors: [] }; instructions?.forEach?.(instruction => { @@ -264,12 +267,18 @@ const findNextStatus = (id: string, bucket: GraphQLProcessBucket): number => { }; const findPreviousStatus = (id: string, bucket: GraphQLProcessBucket): number => { - const status = bucket.statuses.find(status => (status.rest_id ?? status.legacy?.id_str) === id); + const status = bucket.allStatuses.find( + status => (status.rest_id ?? status.legacy?.id_str) === id + ); if (!status) { console.log('uhhh, we could not even find that tweet, dunno how that happened'); return -1; } - return bucket.statuses.findIndex( + if ((status.rest_id ?? status.legacy?.id_str) === status.legacy?.in_reply_to_status_id_str) { + console.log('Tweet does not have a parent'); + return 0; + } + return bucket.allStatuses.findIndex( _status => (_status.rest_id ?? _status.legacy?.id_str) === status.legacy?.in_reply_to_status_id_str ); @@ -308,6 +317,8 @@ export const constructTwitterThread = async ( let response: TweetDetailResult | TweetResultsByRestIdResult | null = null; let status: APITwitterStatus; + + console.log('env', c.env); /* We can use TweetDetail on elongator accounts to increase per-account rate limit. We also use TweetDetail to process threads (WIP) @@ -341,7 +352,7 @@ export const constructTwitterThread = async ( return { status: null, thread: null, author: null, code: 404 }; } - const buildStatus = await buildAPITwitterStatus(c, result, language, false, legacyAPI); + const buildStatus = await buildAPITwitterStatus(c, result, language, null, legacyAPI); if ((buildStatus as FetchResults)?.status === 401) { writeDataPoint(c, language, null, '401'); @@ -372,7 +383,7 @@ export const constructTwitterThread = async ( c, originalStatus, undefined, - false, + null, legacyAPI )) as APITwitterStatus; @@ -390,6 +401,7 @@ export const constructTwitterThread = async ( } const threadStatuses = [originalStatus]; + bucket.allStatuses = bucket.statuses; bucket.statuses = filterBucketStatuses(bucket.statuses, originalStatus); let currentId = id; @@ -465,7 +477,7 @@ export const constructTwitterThread = async ( while (findPreviousStatus(currentId, bucket) !== -1) { const index = findPreviousStatus(currentId, bucket); - const status = bucket.statuses[index]; + const status = bucket.allStatuses[index]; const newCurrentId = status.rest_id ?? status.legacy?.id_str; console.log( @@ -533,10 +545,30 @@ export const constructTwitterThread = async ( code: 200 }; - threadStatuses.forEach(async status => { - socialThread.thread?.push( - (await buildAPITwitterStatus(c, status, undefined, true, false)) as APITwitterStatus - ); + await Promise.all( + threadStatuses.map(async status => { + const builtStatus = (await buildAPITwitterStatus( + c, + status, + undefined, + author, + false + )) as APITwitterStatus; + socialThread.thread?.push(builtStatus); + }) + ); + + // Sort socialThread.thread by id converted to bigint + socialThread.thread?.sort((a, b) => { + const aId = BigInt(a.id); + const bId = BigInt(b.id); + if (aId < bId) { + return -1; + } + if (aId > bId) { + return 1; + } + return 0; }); return socialThread; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index 839a143..05a721e 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -12,7 +12,7 @@ export const buildAPITwitterStatus = async ( c: Context, status: GraphQLTwitterStatus, language: string | undefined, - threadPiece = false, + threadAuthor: null | APIUser, legacyAPI = false // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { @@ -58,26 +58,26 @@ export const buildAPITwitterStatus = async ( apiStatus.text = unescapeText( linkFixer(status.legacy.entities?.urls, status.legacy.full_text || '') ); - if (!threadPiece) { - apiStatus.author = { - id: apiUser.id, - name: apiUser.name, - screen_name: apiUser.screen_name, - avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, - banner_url: apiUser.banner_url, - description: apiUser.description, - location: apiUser.location, - url: apiUser.url, - followers: apiUser.followers, - following: apiUser.following, - joined: apiUser.joined, - statuses: apiUser.statuses, - likes: apiUser.likes, - protected: apiUser.protected, - birthday: apiUser.birthday, - website: apiUser.website - }; - } + // if (threadAuthor && threadAuthor.id !== apiUser.id) { + apiStatus.author = { + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, + avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, + banner_url: apiUser.banner_url, + description: apiUser.description, + location: apiUser.location, + url: apiUser.url, + followers: apiUser.followers, + following: apiUser.following, + joined: apiUser.joined, + statuses: apiUser.statuses, + likes: apiUser.likes, + protected: apiUser.protected, + birthday: apiUser.birthday, + website: apiUser.website + }; + // } apiStatus.replies = status.legacy.reply_count; if (legacyAPI) { // @ts-expect-error Use retweets for legacy API @@ -94,7 +94,9 @@ export const buildAPITwitterStatus = async ( delete apiStatus.author.global_screen_name; } else { apiStatus.reposts = status.legacy.retweet_count; + // if ((threadAuthor && threadAuthor.id !== apiUser.id)) { apiStatus.author.global_screen_name = apiUser.global_screen_name; + // } } apiStatus.likes = status.legacy.favorite_count; apiStatus.embed_card = 'tweet'; @@ -125,6 +127,16 @@ export const buildAPITwitterStatus = async ( apiStatus.is_note_tweet = false; } + if (status.birdwatch_pivot?.subtitle?.text) { + /* We can't automatically replace links because API doesn't give full URLs, only t.co versions :( */ + apiStatus.community_note = { + text: unescapeText(status.birdwatch_pivot?.subtitle?.text ?? ''), + entities: status.birdwatch_pivot.subtitle?.entities ?? [] + }; + } else { + apiStatus.community_note = null; + } + if (status.legacy.lang !== 'unk') { apiStatus.lang = status.legacy.lang; } else { @@ -150,7 +162,7 @@ export const buildAPITwitterStatus = async ( /* We found a quote, let's process that too */ const quote = status.quoted_status_result; if (quote) { - const buildQuote = await buildAPITwitterStatus(c, quote, language, threadPiece, legacyAPI); + const buildQuote = await buildAPITwitterStatus(c, quote, language, threadAuthor, legacyAPI); if ((buildQuote as FetchResults).status) { apiStatus.quote = undefined; } else { @@ -197,7 +209,7 @@ export const buildAPITwitterStatus = async ( */ /* Handle photos and mosaic if available */ - if ((apiStatus?.media.photos?.length || 0) > 1 && !threadPiece) { + if ((apiStatus?.media.photos?.length || 0) > 1 && !threadAuthor) { const mosaic = await handleMosaic(apiStatus.media?.photos || [], id); if (typeof apiStatus.media !== 'undefined' && mosaic !== null) { apiStatus.media.mosaic = mosaic; diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts index 5f29269..dea9c21 100644 --- a/src/realms/twitter/routes/status.ts +++ b/src/realms/twitter/routes/status.ts @@ -3,6 +3,7 @@ import { Constants } from '../../../constants'; import { getBaseRedirectUrl } from '../router'; import { handleStatus } from '../../../embed/status'; import { Strings } from '../../../strings'; +import { Experiment, experimentCheck } from '../../../experiments'; /* Handler for status request */ export const statusRequest = async (c: Context) => { @@ -55,6 +56,13 @@ export const statusRequest = async (c: Context) => { } else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) { console.log('Forced instant view request'); flags.forceInstantView = true; + } else if ( + experimentCheck(Experiment.IV_FORCE_THREAD_UNROLL, userAgent.includes('Telegram')) || + Constants.INSTANT_VIEW_THREADS_DOMAINS.includes(url.hostname) + ) { + console.log('Forced unroll instant view'); + flags.forceInstantView = true; + flags.instantViewUnrollThreads = true; } else if (Constants.GALLERY_DOMAINS.includes(url.hostname)) { console.log('Gallery embed request'); flags.gallery = true; diff --git a/src/render/instantview.ts b/src/render/instantview.ts index 8c8a45b..39735ad 100644 --- a/src/render/instantview.ts +++ b/src/render/instantview.ts @@ -4,6 +4,12 @@ import { getSocialTextIV } from '../helpers/socialproof'; import { sanitizeText } from '../helpers/utils'; import { Strings } from '../strings'; +enum AuthorActionType { + Reply = 'Reply', + Original = 'Original', + FollowUp = 'FollowUp' +} + const populateUserLinks = (status: APIStatus, text: string): string => { /* TODO: Maybe we can add username splices to our API so only genuinely valid users are linked? */ text.match(/@(\w{1,15})/g)?.forEach(match => { @@ -16,7 +22,7 @@ const populateUserLinks = (status: APIStatus, text: string): string => { return text; }; -const generateStatusMedia = (status: APIStatus): string => { +const generateStatusMedia = (status: APIStatus, author: APIUser): string => { let media = ''; if (status.media?.all?.length) { status.media.all.forEach(mediaItem => { @@ -30,10 +36,10 @@ const generateStatusMedia = (status: APIStatus): string => { }); break; case 'video': - media += `