From 0a27b37bd1a995f16251cad7c12721378c68927d Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Wed, 8 Nov 2023 19:49:32 -0500 Subject: [PATCH 01/30] Remove seemingly fixed workaround --- src/worker.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/worker.ts b/src/worker.ts index 95984dd..ebd5588 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -624,8 +624,7 @@ const sentryWrapper = async (event: FetchEvent, test = false): Promise => /* TODO: Figure out what changed between @sentry/integration 7.65.0 and 7.66.0 https://github.com/getsentry/sentry-javascript/compare/7.65.0...7.66.0 which caused types to go apeshit */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - integrations: [new RewriteFrames({ root: '/' }) as any], + integrations: [new RewriteFrames({ root: '/' })], /* event includes 'waitUntil', which is essential for Sentry logs to be delivered. Also includes 'request' -- no need to set it separately. */ release: RELEASE_NAME From 396954a4ae3e0c42c2755ead379609b798d01022 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Wed, 8 Nov 2023 20:45:15 -0500 Subject: [PATCH 02/30] Add hono, move to esm esbuild output --- esbuild.config.mjs | 1 + package-lock.json | 9 +++++++++ package.json | 1 + 3 files changed, 11 insertions(+) diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 82f22d9..ab42ea5 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -62,6 +62,7 @@ await esbuild.build({ outdir: 'dist', minify: true, bundle: true, + format: 'esm', plugins: [ sentryEsbuildPlugin({ org: process.env.SENTRY_ORG, diff --git a/package-lock.json b/package-lock.json index 9c44fd9..27f2ab1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "MIT", "dependencies": { + "hono": "^3.9.2", "itty-router": "^4.0.23", "toucan-js": "^3.3.1" }, @@ -4255,6 +4256,14 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/hono/-/hono-3.9.2.tgz", + "integrity": "sha512-180NOiMadqU3lGmN6ajPDZvZPWus3a9mtVaAUR9uG0SImngBwRLA8vbnV0oUfUAgFT4nX55sGV9dVA06OuikHA==", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", diff --git a/package.json b/package.json index b3e54c0..50b2ccf 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "wrangler": "^3.15.0" }, "dependencies": { + "hono": "^3.9.2", "itty-router": "^4.0.23", "toucan-js": "^3.3.1" } From 01da30b9e8d1cb4d9d52106fd187dd20399b1cca Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 9 Nov 2023 03:53:24 -0500 Subject: [PATCH 03/30] Convert worker to Hono and ES Module --- esbuild.config.mjs | 2 +- package-lock.json | 21 +- package.json | 5 +- src/caches.ts | 79 +++ src/constants.ts | 6 +- src/embed/status.ts | 71 ++- src/fetch.ts | 30 +- src/helpers/graphql.ts | 4 +- src/helpers/translate.ts | 8 +- src/helpers/utils.ts | 1 - src/providers/twitter/conversation.ts | 53 +- src/providers/twitter/processor.ts | 18 +- src/providers/twitter/profile.ts | 5 +- src/realms/api/router.ts | 24 + src/realms/common/version.ts | 29 + src/realms/twitter/router.ts | 58 ++ src/realms/twitter/routes/oembed.ts | 34 ++ src/realms/twitter/routes/profile.ts | 70 +++ src/realms/twitter/routes/redirects.ts | 74 +++ src/realms/twitter/routes/status.ts | 151 ++++++ src/strings.ts | 14 - src/types/types.d.ts | 6 - src/user.ts | 57 +- src/worker.ts | 723 +++---------------------- 24 files changed, 746 insertions(+), 797 deletions(-) create mode 100644 src/caches.ts create mode 100644 src/realms/api/router.ts create mode 100644 src/realms/common/version.ts create mode 100644 src/realms/twitter/router.ts create mode 100644 src/realms/twitter/routes/oembed.ts create mode 100644 src/realms/twitter/routes/profile.ts create mode 100644 src/realms/twitter/routes/redirects.ts create mode 100644 src/realms/twitter/routes/status.ts diff --git a/esbuild.config.mjs b/esbuild.config.mjs index ab42ea5..57ede78 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -60,7 +60,7 @@ await esbuild.build({ entryPoints: ['src/worker.ts'], sourcemap: 'external', outdir: 'dist', - minify: true, + minify: false, bundle: true, format: 'esm', plugins: [ diff --git a/package-lock.json b/package-lock.json index 27f2ab1..6210e29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,8 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "hono": "^3.9.2", - "itty-router": "^4.0.23", - "toucan-js": "^3.3.1" + "@hono/sentry": "^1.0.0", + "hono": "^3.9.2" }, "devDependencies": { "@cloudflare/workers-types": "^4.20231025.0", @@ -1241,6 +1240,17 @@ "node": ">=14" } }, + "node_modules/@hono/sentry": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@hono/sentry/-/sentry-1.0.0.tgz", + "integrity": "sha512-GbPxgpGuasM2zRCSaA77MPWu4KDcuk/EMf7JJykjCvnOTbjmtr7FovNxsvg7xlXCIjZDgLmqBaoJMi3AxbeIAA==", + "dependencies": { + "toucan-js": "^3.2.2" + }, + "peerDependencies": { + "hono": "3.*" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.13", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", @@ -4549,11 +4559,6 @@ "node": ">=8" } }, - "node_modules/itty-router": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/itty-router/-/itty-router-4.0.23.tgz", - "integrity": "sha512-tP1NI8PVK43vWlBnIPqj47ni5FDSczFviA4wgBznscndo8lEvBA+pO3DD1rNbIQPcZhprr775iUTunyGvQMcBw==" - }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", diff --git a/package.json b/package.json index 50b2ccf..51c3e73 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,7 @@ "wrangler": "^3.15.0" }, "dependencies": { - "hono": "^3.9.2", - "itty-router": "^4.0.23", - "toucan-js": "^3.3.1" + "@hono/sentry": "^1.0.0", + "hono": "^3.9.2" } } diff --git a/src/caches.ts b/src/caches.ts new file mode 100644 index 0000000..2b2fb6a --- /dev/null +++ b/src/caches.ts @@ -0,0 +1,79 @@ +import { MiddlewareHandler } from 'hono'; +import { Constants } from './constants'; +import {} from 'hono'; + +/* Wrapper to handle caching, and misc things like catching robots.txt */ +export const cacheMiddleware = (): MiddlewareHandler => async (c, next) => { + const request = c.req; + const userAgent = request.header('User-Agent') ?? ''; + // https://developers.cloudflare.com/workers/examples/cache-api/ + const cacheUrl = new URL( + userAgent.includes('Telegram') + ? `${request.url}&telegram` + : userAgent.includes('Discord') + ? `${request.url}&discord` + : request.url + ); + + console.log('cacheUrl', cacheUrl); + + const cacheKey = new Request(cacheUrl.toString(), request); + const cache = caches.default; + + switch (request.method) { + case 'GET': + if ( + !Constants.API_HOST_LIST.includes(cacheUrl.hostname) && + !request.header('Cookie')?.includes('base_redirect') + ) { + /* cache may be undefined in tests */ + const cachedResponse = await cache.match(cacheKey); + + if (cachedResponse) { + console.log('Cache hit'); + return new Response(cachedResponse.body, cachedResponse); + } + + console.log('Cache miss'); + } + + await next(); + + // eslint-disable-next-line no-case-declarations + const response = c.res.clone(); + + /* Store the fetched response as cacheKey + Use waitUntil so you can return the response without blocking on + writing to cache */ + try { + c.executionCtx.waitUntil(cache.put(cacheKey, response.clone())); + } catch (error) { + console.error((error as Error).stack); + } + + return response; + /* Telegram sends this from Webpage Bot, and Cloudflare sends it if we purge cache, and we respect it. + PURGE is not defined in an RFC, but other servers like Nginx apparently use it. + + Update 2023-11-09: + + For some reason, even before migrating to Hono, this returns 403 Forbidden now when PURGEd. + I'm not sure why, as this is clearly not what we are doing. Is Cloudflare doing this? Is something else wrong? We'll also accept DELETE to do the same I guess. */ + case 'PURGE': + case 'DELETE': + console.log('Purging cache as requested'); + await cache.delete(cacheKey); + return c.text('') + /* yes, we do give HEAD */ + case 'HEAD': + return c.text('') + /* We properly state our OPTIONS when asked */ + case 'OPTIONS': + c.header('allow', Constants.RESPONSE_HEADERS.allow) + c.status(204) + return c.text(''); + default: + c.status(405); + return c.text('') + } +}; \ No newline at end of file diff --git a/src/constants.ts b/src/constants.ts index 0b0bfa5..dada8b7 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -59,8 +59,10 @@ export const Constants = { RESPONSE_HEADERS: { 'allow': 'OPTIONS, GET, PURGE, HEAD', 'content-type': 'text/html;charset=UTF-8', - 'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`, - 'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets + 'x-powered-by': `${RELEASE_NAME}`, + 'x-trans-rights': 'true', + 'cache-control': 'max-age=3600', // Can be overriden in some cases, like unfinished poll tweets + 'Vary': 'Accept-Encoding, User-Agent' }, API_RESPONSE_HEADERS: { 'access-control-allow-origin': '*', diff --git a/src/embed/status.ts b/src/embed/status.ts index 3749e1a..8340bbc 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -7,45 +7,41 @@ import { renderPhoto } from '../render/photo'; import { renderVideo } from '../render/video'; import { renderInstantView } from '../render/instantview'; import { constructTwitterThread } from '../providers/twitter/conversation'; -import { IRequest } from 'itty-router'; +import { Context } from 'hono'; -export const returnError = (error: string): StatusResponse => { - return { - text: Strings.BASE_HTML.format({ - lang: '', - headers: [ - ``, - `` - ].join('') - }) - }; +export const returnError = (c: Context, error: string): Response => { + return c.text(Strings.BASE_HTML.format({ + lang: '', + headers: [ + ``, + `` + ].join('') + })); }; - /* Handler for Twitter statuses (Tweets). Like Twitter, we use the terminologies interchangably. */ export const handleStatus = async ( + c: Context, status: string, mediaNumber: number | undefined, userAgent: string, flags: InputFlags, language: string, - event: FetchEvent, - request: IRequest // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { console.log('Direct?', flags?.direct); let fetchWithThreads = false; /* TODO: Enable actually pulling threads once we can actually do something with them */ - if (request?.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) { + if (c.req.header('user-agent')?.includes('Telegram') && !flags?.direct) { fetchWithThreads = false; } const thread = await constructTwitterThread( status, fetchWithThreads, - request, + c, language, flags?.api ?? false ); @@ -76,27 +72,27 @@ export const handleStatus = async ( /* Catch this request if it's an API response */ if (flags?.api) { - return { - response: new Response(JSON.stringify(api), { - headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }, - status: api.code - }) - }; + c.status(api.code); + // Add every header from Constants.API_RESPONSE_HEADERS + for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { + c.header(header, value); + } + return c.text(JSON.stringify(api)); } if (tweet === null) { - return returnError(Strings.ERROR_TWEET_NOT_FOUND); + return returnError(c, Strings.ERROR_TWEET_NOT_FOUND); } /* If there was any errors fetching the Tweet, we'll return it */ switch (api.code) { case 401: - return returnError(Strings.ERROR_PRIVATE); + return returnError(c, Strings.ERROR_PRIVATE); case 404: - return returnError(Strings.ERROR_TWEET_NOT_FOUND); + return returnError(c, Strings.ERROR_TWEET_NOT_FOUND); case 500: console.log(api); - return returnError(Strings.ERROR_API_FAIL); + return returnError(c, Strings.ERROR_API_FAIL); } const isTelegram = (userAgent || '').indexOf('Telegram') > -1; @@ -146,7 +142,7 @@ export const handleStatus = async ( } if (redirectUrl) { - return { response: Response.redirect(redirectUrl, 302) }; + return c.redirect(redirectUrl, 302); } } @@ -157,7 +153,6 @@ export const handleStatus = async ( const engagementText = authorText.replace(/ {4}/g, ' '); let siteName = Constants.BRANDING_NAME; let newText = tweet.text; - let cacheControl: string | null = null; /* Base headers included in all responses */ const headers = [ @@ -217,7 +212,8 @@ export const handleStatus = async ( console.log('overrideMedia', JSON.stringify(overrideMedia)); if (!flags?.textOnly) { - const media = tweet.media?.all && tweet.media?.all.length > 0 ? tweet.media : tweet.quote?.media || {} + const media = + tweet.media?.all && tweet.media?.all.length > 0 ? tweet.media : tweet.quote?.media || {}; if (overrideMedia) { let instructions: ResponseInstructions; @@ -340,7 +336,7 @@ export const handleStatus = async ( Yes, checking if this is a string is a hacky way to do this, but it can do it in way less code than actually comparing dates */ if (poll.time_left_en !== 'Final results') { - cacheControl = Constants.POLL_TWEET_CACHE; + c.header('cache-control', Constants.POLL_TWEET_CACHE); } /* And now we'll put the poll right after the Tweet text! */ @@ -427,12 +423,9 @@ export const handleStatus = async ( const lang = tweet.lang === null ? 'en' : tweet.lang || 'en'; /* Finally, after all that work we return the response HTML! */ - return { - text: Strings.BASE_HTML.format({ - lang: `lang="${lang}"`, - headers: headers.join(''), - body: ivbody - }).replace(/>(\s+)<'), // Remove whitespace between tags - cacheControl: cacheControl - }; + return c.text(Strings.BASE_HTML.format({ + lang: `lang="${lang}"`, + headers: headers.join(''), + body: ivbody + }).replace(/>(\s+)<')); }; diff --git a/src/fetch.ts b/src/fetch.ts index 5973279..1ab32d8 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,3 +1,4 @@ +import { Context } from 'hono'; import { Constants } from './constants'; import { Experiment, experimentCheck } from './experiments'; import { generateUserAgent } from './helpers/useragent'; @@ -19,11 +20,11 @@ const generateSnowflake = () => { globalThis.fetchCompletedTime = 0; export const twitterFetch = async ( + c: Context, url: string, - event: FetchEvent, useElongator = experimentCheck( Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' + typeof c.env.TwitterProxy !== 'undefined' ), validateFunction: (response: unknown) => boolean, elongatorRequired = false @@ -140,10 +141,10 @@ export const twitterFetch = async ( let apiRequest; try { - if (useElongator && typeof TwitterProxy !== 'undefined') { + if (useElongator && typeof c.env.TwitterProxy !== 'undefined') { console.log('Fetching using elongator'); const performanceStart = performance.now(); - apiRequest = await TwitterProxy.fetch(url, { + apiRequest = await c.env.TwitterProxy.fetch(url, { method: 'GET', headers: headers }); @@ -170,8 +171,8 @@ export const twitterFetch = async ( return {}; } !useElongator && - event && - event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); + c.executionCtx && + c.executionCtx.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); if (useElongator) { console.log('Elongator request failed, trying again without it'); wasElongatorDisabled = true; @@ -186,7 +187,7 @@ export const twitterFetch = async ( if ( !wasElongatorDisabled && !useElongator && - typeof TwitterProxy !== 'undefined' && + typeof c.env.TwitterProxy !== 'undefined' && (response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason === 'NsfwLoggedOut' ) { @@ -200,8 +201,8 @@ export const twitterFetch = async ( /* Running out of requests within our rate limit, let's purge the cache */ if (!useElongator && remainingRateLimit < 10) { console.log(`Purging token on this edge due to low rate limit remaining`); - event && - event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); + c.executionCtx && + c.executionCtx.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); } if (!validateFunction(response)) { @@ -219,7 +220,7 @@ export const twitterFetch = async ( continue; } /* If we've generated a new token, we'll cache it */ - if (event && newTokenGenerated && activate) { + if (c.executionCtx && newTokenGenerated && activate) { const cachingResponse = new Response(await activate.clone().text(), { headers: { ...tokenHeaders, @@ -227,7 +228,7 @@ export const twitterFetch = async ( } }); console.log('Caching guest token'); - event.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); + c.executionCtx.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); } // @ts-expect-error - We'll pin the guest token to whatever response we have @@ -243,13 +244,13 @@ export const twitterFetch = async ( export const fetchUser = async ( username: string, - event: FetchEvent, + c: Context, useElongator = experimentCheck( Experiment.ELONGATOR_PROFILE_API, - typeof TwitterProxy !== 'undefined' + typeof c.env.TwitterProxy !== 'undefined' ) ): Promise => { - return (await twitterFetch( + return (await twitterFetch(c, `${ Constants.TWITTER_ROOT }/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?variables=${encodeURIComponent( @@ -266,7 +267,6 @@ export const fetchUser = async ( verified_phone_label_enabled: true }) )}`, - event, useElongator, // Validator function (_res: unknown) => { diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index e11097e..310dd75 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -17,6 +17,8 @@ export const isGraphQLTweet = (response: unknown): response is GraphQLTweet => { typeof response === 'object' && response !== null && // @ts-expect-error it's 6 am please let me sleep - ('__typename' in response && (response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults') || ('legacy' in response && response.legacy?.full_text)) + (('__typename' in response && + (response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults')) || + ('legacy' in response && response.legacy?.full_text)) ); }; diff --git a/src/helpers/translate.ts b/src/helpers/translate.ts index a4f619e..fcdc22d 100644 --- a/src/helpers/translate.ts +++ b/src/helpers/translate.ts @@ -1,10 +1,12 @@ +import { Context } from 'hono'; import { Constants } from '../constants'; /* Handles translating Tweets when asked! */ export const translateTweet = async ( tweet: GraphQLTweet, guestToken: string, - language: string + language: string, + c: Context ): Promise => { const csrfToken = crypto.randomUUID().replace(/-/g, ''); // Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match @@ -44,14 +46,14 @@ export const translateTweet = async ( headers['x-twitter-client-language'] = language; /* As of August 2023, you can no longer fetch translations with guest token */ - if (typeof TwitterProxy === 'undefined') { + if (typeof c.env.TwitterProxy === 'undefined') { return null; } try { const url = `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`; console.log(url, headers); - translationApiResponse = await TwitterProxy.fetch(url, { + translationApiResponse = await c.env.TwitterProxy.fetch(url, { method: 'GET', headers: headers }); diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index 6b86c01..fc04bfd 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -32,7 +32,6 @@ export const truncateWithEllipsis = (str: string, maxLength: number): string => return truncated.length < str.length ? truncated + '…' : truncated; }; - const numberFormat = new Intl.NumberFormat('en-US'); export const formatNumber = (num: number) => numberFormat.format(num); diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 59381ff..4e1473f 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -1,17 +1,17 @@ -import { IRequest } from 'itty-router'; import { Constants } from '../../constants'; import { twitterFetch } from '../../fetch'; import { buildAPITweet } from './processor'; import { Experiment, experimentCheck } from '../../experiments'; import { isGraphQLTweet } from '../../helpers/graphql'; +import { Context } from 'hono'; export const fetchTweetDetail = async ( + c: Context, status: string, - event: FetchEvent, - useElongator = typeof TwitterProxy !== 'undefined', + useElongator = typeof c.env.TwitterProxy !== 'undefined', cursor: string | null = null ): Promise => { - return (await twitterFetch( + return (await twitterFetch(c, `${ Constants.TWITTER_ROOT }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( @@ -56,7 +56,6 @@ export const fetchTweetDetail = async ( withArticleRichContentState: true }) )}`, - event, useElongator, (_conversation: unknown) => { const conversation = _conversation as TweetDetailResult; @@ -90,13 +89,13 @@ export const fetchTweetDetail = async ( export const fetchByRestId = async ( status: string, - event: FetchEvent, + c: Context, useElongator = experimentCheck( Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' + typeof c.env.TwitterProxy !== 'undefined' ) ): Promise => { - return (await twitterFetch( + return (await twitterFetch(c, `${ Constants.TWITTER_ROOT }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( @@ -133,7 +132,6 @@ export const fetchByRestId = async ( withArticleRichContentState: true }) )}`, - event, useElongator, (_conversation: unknown) => { const conversation = _conversation as TweetResultsByRestIdResult; @@ -273,7 +271,7 @@ const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { export const constructTwitterThread = async ( id: string, processThread = false, - request: IRequest, + c: Context, language: string | undefined, legacyAPI = false ): Promise => { @@ -286,9 +284,13 @@ export const constructTwitterThread = async ( Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why. I'll figure out why eventually, but for now just don't use TweetDetail for this. */ - if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { + if ( + typeof c.env.TwitterProxy !== 'undefined' && + !language && + (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread) + ) { console.log('Using TweetDetail for request...'); - response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult; + response = (await fetchTweetDetail(c, id)) as TweetDetailResult; console.log(response); @@ -312,7 +314,7 @@ export const constructTwitterThread = async ( /* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */ if (!response) { console.log('Using TweetResultsByRestId for request...'); - response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult; + response = (await fetchByRestId(id, c)) as TweetResultsByRestIdResult; const result = response?.data?.tweetResult?.result as GraphQLTweet; @@ -320,7 +322,7 @@ export const constructTwitterThread = async ( return { post: null, thread: null, author: null, code: 404 }; } - const buildPost = await buildAPITweet(result, language, false, legacyAPI); + const buildPost = await buildAPITweet(c, result, language, false, legacyAPI); if ((buildPost as FetchResults)?.status === 401) { return { post: null, thread: null, author: null, code: 401 }; @@ -343,7 +345,7 @@ export const constructTwitterThread = async ( return { post: null, thread: null, author: null, code: 404 }; } - post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet; + post = (await buildAPITweet(c, originalTweet, undefined, false, legacyAPI)) as APITweet; if (post === null) { return { post: null, thread: null, author: null, code: 404 }; @@ -400,7 +402,7 @@ export const constructTwitterThread = async ( let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); + loadCursor = await fetchTweetDetail(c, id, true, cursor.value); if ( typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === @@ -464,7 +466,7 @@ export const constructTwitterThread = async ( let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); + loadCursor = await fetchTweetDetail(c, id, true, cursor.value); if ( typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === @@ -501,18 +503,21 @@ export const constructTwitterThread = async ( }; threadTweets.forEach(async tweet => { - socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet); + socialThread.thread?.push((await buildAPITweet(c, tweet, undefined, true, false)) as APITweet); }); return socialThread; }; -export const threadAPIProvider = async (request: IRequest) => { - const { id } = request.params; +export const threadAPIProvider = async (c: Context) => { + const id = c.req.param('id') as string; - const processedResponse = await constructTwitterThread(id, true, request, undefined); + const processedResponse = await constructTwitterThread(id, true, c, undefined); - return new Response(JSON.stringify(processedResponse), { - headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } - }); + c.status(processedResponse.code); + // Add every header from Constants.API_RESPONSE_HEADERS + for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { + c.header(header, value); + } + return c.text(JSON.stringify(processedResponse)); }; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index 62e7b76..be43668 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -7,12 +7,14 @@ import { unescapeText } from '../../helpers/utils'; import { processMedia } from '../../helpers/media'; import { convertToApiUser } from './profile'; import { translateTweet } from '../../helpers/translate'; +import { Context } from 'hono'; export const buildAPITweet = async ( + c: Context, tweet: GraphQLTweet, language: string | undefined, threadPiece = false, - legacyAPI = false + legacyAPI = false, // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { const apiTweet = {} as APITweet; @@ -149,19 +151,13 @@ export const buildAPITweet = async ( /* We found a quote tweet, let's process that too */ const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { - const buildQuote = (await buildAPITweet( - quoteTweet, - language, - threadPiece, - legacyAPI - )); + const buildQuote = await buildAPITweet(c, quoteTweet, language, threadPiece, legacyAPI); if ((buildQuote as FetchResults).status) { - apiTweet.quote = undefined + apiTweet.quote = undefined; } else { apiTweet.quote = buildQuote as APITweet; } - /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ if (apiTweet.embed_card === 'tweet' && typeof apiTweet.quote !== 'undefined') { apiTweet.embed_card = apiTweet.quote.embed_card; @@ -237,12 +233,12 @@ export const buildAPITweet = async ( apiTweet.embed_card = 'player'; } - console.log('language?', language) + console.log('language?', language); /* If a language is specified in API or by user, let's try translating it! */ if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { console.log(`Attempting to translate Tweet to ${language}...`); - const translateAPI = await translateTweet(tweet, '', language); + const translateAPI = await translateTweet(tweet, '', language, c); if (translateAPI !== null && translateAPI?.translation) { apiTweet.translation = { text: unescapeText( diff --git a/src/providers/twitter/profile.ts b/src/providers/twitter/profile.ts index fffb15f..0f4ad81 100644 --- a/src/providers/twitter/profile.ts +++ b/src/providers/twitter/profile.ts @@ -1,3 +1,4 @@ +import { Context } from 'hono'; import { Constants } from '../../constants'; import { fetchUser } from '../../fetch'; import { linkFixer } from '../../helpers/linkFixer'; @@ -79,10 +80,10 @@ const populateUserProperties = async ( available for free using api.fxtwitter.com. */ export const userAPI = async ( username: string, - event: FetchEvent + c: Context // flags?: InputFlags ): Promise => { - const userResponse = await fetchUser(username, event); + const userResponse = await fetchUser(username, c); if (!userResponse || !Object.keys(userResponse).length) { return { code: 404, diff --git a/src/realms/api/router.ts b/src/realms/api/router.ts new file mode 100644 index 0000000..365ce6a --- /dev/null +++ b/src/realms/api/router.ts @@ -0,0 +1,24 @@ +import { Hono } from 'hono'; +import { statusRequest } from '../twitter/routes/status'; +import { profileRequest } from '../twitter/routes/profile'; +import { Strings } from '../../strings'; + +export const api = new Hono(); + +/* Current v1 API endpoints. Currently, these still go through the Twitter embed requests. API v2+ won't do this. */ +api.get('/:handle?/status/:id/:language?', statusRequest); +api.get( + '/:handle?/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + statusRequest +); + +api.get('/:handle', profileRequest); + +api.get( + '/robots.txt', + async (c) => { + c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); + c.status(200); + return c.text(Strings.ROBOTS_TXT); + } +); diff --git a/src/realms/common/version.ts b/src/realms/common/version.ts new file mode 100644 index 0000000..0555431 --- /dev/null +++ b/src/realms/common/version.ts @@ -0,0 +1,29 @@ +import { Context } from 'hono'; +import { Constants } from '../../constants'; +import { sanitizeText } from '../../helpers/utils'; +import { Strings } from '../../strings'; + +export const versionRoute = async (context: Context) => { + const request = context.req; + return new Response( + Strings.VERSION_HTML.format({ + rtt: request.raw.cf?.clientTcpRtt ? `🏓 ${request.raw.cf.clientTcpRtt} ms RTT` : '', + colo: (request.raw.cf?.colo as string) ?? '??', + httpversion: (request.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', + tlsversion: (request.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version', + ip: request.header('x-real-ip') ?? request.header('cf-connecting-ip') ?? 'Unknown IP', + city: (request.raw.cf?.city as string) ?? 'Unknown City', + region: (request.raw.cf?.region as string) ?? request.raw.cf?.country ?? 'Unknown Region', + country: (request.raw.cf?.country as string) ?? 'Unknown Country', + asn: `AS${request.raw.cf?.asn ?? '??'} (${request.raw.cf?.asOrganization ?? 'Unknown ASN'})`, + ua: sanitizeText(request.header('user-agent') ?? 'Unknown User Agent') + }), + { + headers: { + ...Constants.RESPONSE_HEADERS, + 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate' + }, + status: 200 + } + ); +}; diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts new file mode 100644 index 0000000..c19ad00 --- /dev/null +++ b/src/realms/twitter/router.ts @@ -0,0 +1,58 @@ +import { Context, Hono } from 'hono'; +// import { cache } from "hono/cache"; +import { versionRoute } from '../common/version'; +import { Strings } from '../../strings'; +import { Constants } from '../../constants'; +import { genericTwitterRedirect, setRedirectRequest } from './routes/redirects'; +import { profileRequest } from './routes/profile'; +import { statusRequest } from './routes/status'; +import { oembed } from './routes/oembed'; + +export const twitter = new Hono(); + +twitter.get('/status/:id', statusRequest); +// twitter.get('/:handle/status/:id', statusRequest); +// twitter.get('/:prefix/:handle/status/:id/:language?', statusRequest); +// twitter.get( +// '/:prefix/:handle/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', +// statusRequest +// ); +// twitter.get('/:handle?/:endpoint{status(es)?}/:id/:language?', statusRequest); +// twitter.get( +// '/:handle?/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', +// statusRequest +// ); + +twitter.get('/version', versionRoute); +twitter.get('/set_base_redirect', setRedirectRequest); +twitter.get('/oembed', oembed); + +twitter.get( + '/robots.txt', + async (c) => { + c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); + c.status(200); + return c.text(Strings.ROBOTS_TXT); + } +); + +twitter.get('/i/events/:id', genericTwitterRedirect); +twitter.get('/hashtag/:hashtag', genericTwitterRedirect); + +twitter.get('/:handle', profileRequest); + +export const getBaseRedirectUrl = (c: Context) => { + const baseRedirect = c.req.header('cookie')?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0]; + + if (baseRedirect) { + console.log('Found base redirect', baseRedirect); + try { + new URL(baseRedirect); + } catch (e) { + return Constants.TWITTER_ROOT; + } + return baseRedirect.endsWith('/') ? baseRedirect.slice(0, -1) : baseRedirect; + } + + return Constants.TWITTER_ROOT; +}; diff --git a/src/realms/twitter/routes/oembed.ts b/src/realms/twitter/routes/oembed.ts new file mode 100644 index 0000000..cd5698f --- /dev/null +++ b/src/realms/twitter/routes/oembed.ts @@ -0,0 +1,34 @@ +import { Context } from 'hono'; +import motd from '../../../../motd.json'; +import { Constants } from '../../../constants'; +import { Strings } from '../../../strings'; + +/* Yes, I actually made the endpoint /owoembed. Deal with it. */ +export const oembed = async (c: Context) => { + console.log('oembed hit!'); + const { searchParams } = new URL(c.req.url); + + /* Fallbacks */ + const text = searchParams.get('text') || 'Twitter'; + const author = searchParams.get('author') || 'jack'; + const status = searchParams.get('status') || '20'; + + const random = Math.floor(Math.random() * Object.keys(motd).length); + const [name, url] = Object.entries(motd)[random]; + + const test = { + author_name: text, + author_url: `${Constants.TWITTER_ROOT}/${encodeURIComponent(author)}/status/${status}`, + /* Change provider name if tweet is on deprecated domain. */ + provider_name: + searchParams.get('deprecated') === 'true' ? Strings.DEPRECATED_DOMAIN_NOTICE_DISCORD : name, + provider_url: url, + title: Strings.DEFAULT_AUTHOR_TEXT, + type: 'link', + version: '1.0' + }; + c.header('content-type', 'application/json') + c.status(200); + /* Stringify and send it on its way! */ + return c.text(JSON.stringify(test)) +}; diff --git a/src/realms/twitter/routes/profile.ts b/src/realms/twitter/routes/profile.ts new file mode 100644 index 0000000..78de44c --- /dev/null +++ b/src/realms/twitter/routes/profile.ts @@ -0,0 +1,70 @@ +import { Context } from 'hono'; +import { Constants } from '../../../constants'; +import { handleProfile } from '../../../user'; +import { getBaseRedirectUrl } from '../router'; + +/* Handler for User Profiles */ +export const profileRequest = async (c: Context) => { + const handle = c.req.param('handle'); + const url = new URL(c.req.url); + const userAgent = c.req.header('User-Agent') || ''; + const flags: InputFlags = {}; + + /* User Agent matching for embed generators, bots, crawlers, and other automated + tools. It's pretty all-encompassing. Note that Firefox/92 is in here because + Discord sometimes uses the following UA: + + Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 + + I'm not sure why that specific one, it's pretty weird, but this edge case ensures + stuff keeps working. + + On the very rare off chance someone happens to be using specifically Firefox 92, + the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ + const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null; + + /* If not a valid screen name, we redirect to project GitHub */ + if (handle.match(/\w{1,15}/gi)?.[0] !== handle) { + return c.redirect(Constants.REDIRECT_URL, 302); + } + const username = handle.match(/\w{1,15}/gi)?.[0] as string; + /* Check if request is to api.fxtwitter.com */ + if (Constants.API_HOST_LIST.includes(url.hostname)) { + console.log('JSON API request'); + flags.api = true; + } + + const baseUrl = getBaseRedirectUrl(c); + + /* Do not cache if using a custom redirect */ + const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; + + if (cacheControl) { + c.header('cache-control', cacheControl); + } + /* Direct media or API access bypasses bot check, returning same response regardless of UA */ + if (isBotUA || flags.api) { + if (isBotUA) { + console.log(`Matched bot UA ${userAgent}`); + } else { + console.log('Bypass bot check'); + } + + /* This throws the necessary data to handleStatus (in status.ts) */ + const profileResponse = await handleProfile(c, username, flags); + /* Check for custom redirect */ + + if (!isBotUA) { + return c.redirect(`${baseUrl}/${handle}`, 302); + } + + /* Return the response containing embed information */ + return profileResponse; + } else { + /* A human has clicked a fxtwitter.com/:screen_name link! + Obviously we just need to redirect to the user directly.*/ + console.log('Matched human UA', userAgent); + + return c.redirect(`${baseUrl}/${handle}`, 302) + } +}; diff --git a/src/realms/twitter/routes/redirects.ts b/src/realms/twitter/routes/redirects.ts new file mode 100644 index 0000000..f026d01 --- /dev/null +++ b/src/realms/twitter/routes/redirects.ts @@ -0,0 +1,74 @@ +import { Context } from 'hono'; +import { Strings } from '../../../strings'; +import { sanitizeText } from '../../../helpers/utils'; +import { getBaseRedirectUrl } from '../router'; +import { Constants } from '../../../constants'; + +export const genericTwitterRedirect = async (c: Context) => { + const url = new URL(c.req.url); + const baseUrl = getBaseRedirectUrl(c); + /* Do not cache if using a custom redirect */ + const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; + + if (cacheControl) { + c.header('cache-control', cacheControl); + } + + return c.redirect(`${baseUrl}${url.pathname}`, 302); +}; + +export const setRedirectRequest = async (c: Context) => { + /* Query params */ + const { searchParams } = new URL(c.req.url); + let url = searchParams.get('url'); + + /* Check that origin either does not exist or is in our domain list */ + const origin = c.req.header('origin'); + if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) { + c.status(403); + + return c.text(Strings.MESSAGE_HTML.format({ + message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` + })) + } + + if (!url) { + + /* Remove redirect URL */ + // eslint-disable-next-line sonarjs/no-duplicate-string + c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`); + // eslint-disable-next-line sonarjs/no-duplicate-string + c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); + c.status(200); + return c.text(Strings.MESSAGE_HTML.format({ + message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` + })) + } + + try { + new URL(url); + } catch (e) { + try { + new URL(`https://${url}`); + } catch (e) { + /* URL is not well-formed, remove */ + console.log('Invalid base redirect URL, removing cookie before redirect'); + + c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`); + c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); + c.status(200); + return c.text(Strings.MESSAGE_HTML.format({ + message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` + })) + } + + url = `https://${url}`; + } + + c.header('set-cookie', `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`); + c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); + + return c.text(Strings.MESSAGE_HTML.format({ + message: `Successfully set base redirect, you will now be redirected to ${sanitizeText(url)} rather than ${Constants.TWITTER_ROOT}` + })) +}; diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts new file mode 100644 index 0000000..50e07e2 --- /dev/null +++ b/src/realms/twitter/routes/status.ts @@ -0,0 +1,151 @@ +import { Context } from 'hono'; +import { Constants } from '../../../constants'; +import { getBaseRedirectUrl } from '../router'; +import { handleStatus } from '../../../embed/status'; +import { Strings } from '../../../strings'; + +/* Handler for status (Tweet) request */ +export const statusRequest = async (c: Context) => { + const handle = c.req.param('handle'); + const id = c.req.param('id'); + const mediaNumber = c.req.param('mediaNumber'); + const language = c.req.param('language'); + const prefix = c.req.param('prefix'); + const url = new URL(c.req.url); + const flags: InputFlags = {}; + + // eslint-disable-next-line sonarjs/no-duplicate-string + const userAgent = c.req.header('User-Agent') || ''; + + /* Let's return our HTML version for wayback machine (we can add other archivers too in future) */ + if ( + ['archive.org', 'Wayback Machine'].some(service => c.req.header('Via')?.includes?.(service)) + ) { + console.log('Request from archive.org'); + flags.archive = true; + } + + /* User Agent matching for embed generators, bots, crawlers, and other automated + tools. It's pretty all-encompassing. Note that Firefox/92 is in here because + Discord sometimes uses the following UA: + + Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 + + I'm not sure why that specific one, it's pretty weird, but this edge case ensures + stuff keeps working. + + On the very rare off chance someone happens to be using specifically Firefox 92, + the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ + const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null || flags?.archive; + + /* Check if domain is a direct media domain (i.e. d.fxtwitter.com), + the tweet is prefixed with /dl/ or /dir/ (for TwitFix interop), or the + tweet ends in .mp4, .jpg, .jpeg, or .png + + Note that .png is not documented because images always redirect to a jpg, + but it will help someone who does it mistakenly on something like Discord + + Also note that all we're doing here is setting the direct flag. If someone + links a video and ends it with .jpg, it will still redirect to a .mp4! */ + if (url.pathname.match(/\/status(es)?\/\d{2,20}\.(mp4|png|jpe?g)/g)) { + console.log('Direct media request by extension'); + flags.direct = true; + } else if (Constants.DIRECT_MEDIA_DOMAINS.includes(url.hostname)) { + console.log('Direct media request by domain'); + flags.direct = true; + } else if (Constants.TEXT_ONLY_DOMAINS.includes(url.hostname)) { + console.log('Text-only embed request'); + flags.textOnly = true; + } else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) { + console.log('Forced instant view request'); + flags.forceInstantView = true; + } else if (prefix === 'dl' || prefix === 'dir') { + console.log('Direct media request by path prefix'); + flags.direct = true; + } + + /* The pxtwitter.com domain is deprecated and Tweets posted after deprecation + date will have a notice saying we've moved to fxtwitter.com! */ + if ( + Constants.DEPRECATED_DOMAIN_LIST.includes(url.hostname) && + BigInt(id?.match(/\d{2,20}/g)?.[0] || 0) > Constants.DEPRECATED_DOMAIN_EPOCH + ) { + console.log('Request to deprecated domain'); + flags.deprecated = true; + } + + /* TODO: Figure out what we're doing with FixTweet / FixupX branding in future */ + if (/fixup/g.test(url.href)) { + console.log(`We're using x domain`); + flags.isXDomain = true; + } else { + console.log(`We're using twitter domain`); + } + + const baseUrl = getBaseRedirectUrl(c); + + /* Check if request is to api.fxtwitter.com, or the tweet is appended with .json + Note that unlike TwitFix, FixTweet will never generate embeds for .json, and + in fact we only support .json because it's what people using TwitFix API would + be used to. */ + if ( + url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null || + Constants.API_HOST_LIST.includes(url.hostname) + ) { + console.log('JSON API request'); + flags.api = true; + } + + /* Direct media or API access bypasses bot check, returning same response regardless of UA */ + if (isBotUA || flags.direct || flags.api) { + if (isBotUA) { + console.log(`Matched bot UA ${userAgent}`); + } else { + console.log('Bypass bot check'); + } + + /* This throws the necessary data to handleStatus (in status.ts) */ + const statusResponse = await handleStatus(c, + id?.match(/\d{2,20}/)?.[0] || '0', + mediaNumber ? parseInt(mediaNumber) : undefined, + userAgent, + flags, + language + ); + /* Do not cache if using a custom redirect */ + const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; + + if (cacheControl) { + // eslint-disable-next-line sonarjs/no-duplicate-string + c.header('cache-control', cacheControl); + } + + if (statusResponse) { + /* We're checking if the User Agent is a bot again specifically in case they requested + direct media (d.fxtwitter.com, .mp4/.jpg, etc) but the Tweet contains no media. + + Since we obviously have no media to give the user, we'll just redirect to the Tweet. + Embeds will return as usual to bots as if direct media was never specified. */ + if (!isBotUA && !flags.api) { + const baseUrl = getBaseRedirectUrl(c); + + return c.redirect(`${baseUrl}/${handle || 'i'}/status/${id}`, 302); + } + + c.status(200); + /* Return the response containing embed information */ + return statusResponse; + } else { + /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ + c.status(500); + return c.text(Strings.ERROR_UNKNOWN); + } + } else { + /* A human has clicked a fxtwitter.com/:screen_name/status/:id link! + Obviously we just need to redirect to the Tweet directly.*/ + console.log('Matched human UA', userAgent); + + c.redirect(`${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, 302); + return c.text(''); + } +}; diff --git a/src/strings.ts b/src/strings.ts index 3484b95..98c3eaa 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -203,20 +203,6 @@ This is caused by Twitter API downtime or a new bug. Try again in a little while # Check out the docs at https://${API_HOST_LIST.split(',')[0]} to learn how to use it -# ________________ -# / /| -# / / | -# /_______________/ | -# | ___________ | / | -# | | | | / | -# | | | | / | -# | | gaming | | / | -# | |__________| | / | -# | | / | -# | _____ | / | -# | _____________ | / -# |_____________| / - # Good luck, have fun and try not to take over the world! # Instructions below are for robots only, beep boop diff --git a/src/types/types.d.ts b/src/types/types.d.ts index b3465e6..e5eb0d2 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -37,12 +37,6 @@ interface RenderProperties { flags?: InputFlags; } -interface Request { - params: { - [param: string]: string; - }; -} - interface TweetAPIResponse { code: number; message: string; diff --git a/src/user.ts b/src/user.ts index bf976cb..2ea3dda 100644 --- a/src/user.ts +++ b/src/user.ts @@ -1,50 +1,48 @@ +import { Context } from 'hono'; import { Constants } from './constants'; import { Strings } from './strings'; import { userAPI } from './providers/twitter/profile'; -export const returnError = (error: string): StatusResponse => { - return { - text: Strings.BASE_HTML.format({ - lang: '', - headers: [ - ``, - `` - ].join('') - }) - }; +export const returnError = (c: Context, error: string): Response => { + return c.text(Strings.BASE_HTML.format({ + lang: '', + headers: [ + ``, + `` + ].join('') + })); }; /* Handler for Twitter users */ export const handleProfile = async ( + c: Context, username: string, - userAgent?: string, - flags?: InputFlags, - event?: FetchEvent -): Promise => { + flags: InputFlags, +): Promise => { console.log('Direct?', flags?.direct); - const api = await userAPI(username, event as FetchEvent); + const api = await userAPI(username, c); const user = api?.user as APIUser; /* Catch this request if it's an API response */ // For now we just always return the API response while testing if (flags?.api) { - return { - response: new Response(JSON.stringify(api), { - headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }, - status: api.code - }) - }; + c.status(api.code); + // Add every header from Constants.API_RESPONSE_HEADERS + for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { + c.header(header, value); + } + return c.text(JSON.stringify(api)); } /* If there was any errors fetching the User, we'll return it */ switch (api.code) { case 401: - return returnError(Strings.ERROR_PRIVATE); + return returnError(c, Strings.ERROR_PRIVATE); case 404: - return returnError(Strings.ERROR_USER_NOT_FOUND); + return returnError(c, Strings.ERROR_USER_NOT_FOUND); case 500: - return returnError(Strings.ERROR_API_FAIL); + return returnError(c, Strings.ERROR_API_FAIL); } /* Base headers included in all responses */ @@ -53,11 +51,8 @@ export const handleProfile = async ( // TODO Add card creation logic here /* Finally, after all that work we return the response HTML! */ - return { - text: Strings.BASE_HTML.format({ - lang: `lang="en"`, - headers: headers.join('') - }), - cacheControl: null - }; + return c.text(Strings.BASE_HTML.format({ + lang: `lang="en"`, + headers: headers.join('') + })); }; diff --git a/src/worker.ts b/src/worker.ts index ebd5588..d4f7e76 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,663 +1,114 @@ -/* eslint-disable no-case-declarations */ -import { Toucan } from 'toucan-js'; +import { Env, Hono } from 'hono'; +import { timing } from 'hono/timing'; +import { logger } from 'hono/logger'; import { RewriteFrames } from '@sentry/integrations'; - -import { IRequest, Router } from 'itty-router'; -import { Constants } from './constants'; -import { handleStatus } from './embed/status'; +import { sentry } from '@hono/sentry'; import { Strings } from './strings'; +import { Constants } from './constants'; +import { api } from './realms/api/router'; +import { twitter } from './realms/twitter/router'; +import { cacheMiddleware } from './caches'; -import motd from '../motd.json'; -import { sanitizeText } from './helpers/utils'; -import { handleProfile } from './user'; -// import { threadAPIProvider } from './providers/twitter/conversation'; +const noCache = 'max-age=0, no-cache, no-store, must-revalidate'; -declare const globalThis: { - fetchCompletedTime: number; -}; +const app = new Hono<{ Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: AnalyticsEngineDataset } }>({ + getPath: req => { + let url: URL; -const router = Router(); - -const getBaseRedirectUrl = (request: IRequest) => { - const baseRedirect = request.headers - ?.get('cookie') - ?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0]; - - if (baseRedirect) { - console.log('Found base redirect', baseRedirect); try { - new URL(baseRedirect); - } catch (e) { - return Constants.TWITTER_ROOT; + url = new URL(req.url); + } catch (err) { + return '/error'; } - return baseRedirect.endsWith('/') ? baseRedirect.slice(0, -1) : baseRedirect; - } - - return Constants.TWITTER_ROOT; -}; - -/* Handler for status (Tweet) request */ -const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputFlags = {}) => { - const { handle, id, mediaNumber, language, prefix } = request.params; - const url = new URL(request.url); - // eslint-disable-next-line sonarjs/no-duplicate-string - const userAgent = request.headers.get('User-Agent') || ''; - - /* Let's return our HTML version for wayback machine (we can add other archivers too in future) */ - if ( - ['archive.org', 'Wayback Machine'].some( - service => request.headers.get('Via')?.includes?.(service) - ) - ) { - console.log('Request from archive.org'); - flags.archive = true; - } - - /* User Agent matching for embed generators, bots, crawlers, and other automated - tools. It's pretty all-encompassing. Note that Firefox/92 is in here because - Discord sometimes uses the following UA: - - Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 - - I'm not sure why that specific one, it's pretty weird, but this edge case ensures - stuff keeps working. - - On the very rare off chance someone happens to be using specifically Firefox 92, - the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ - const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null || flags?.archive; - - /* Check if domain is a direct media domain (i.e. d.fxtwitter.com), - the tweet is prefixed with /dl/ or /dir/ (for TwitFix interop), or the - tweet ends in .mp4, .jpg, .jpeg, or .png - - Note that .png is not documented because images always redirect to a jpg, - but it will help someone who does it mistakenly on something like Discord - - Also note that all we're doing here is setting the direct flag. If someone - links a video and ends it with .jpg, it will still redirect to a .mp4! */ - if (url.pathname.match(/\/status(es)?\/\d{2,20}\.(mp4|png|jpe?g)/g)) { - console.log('Direct media request by extension'); - flags.direct = true; - } else if (Constants.DIRECT_MEDIA_DOMAINS.includes(url.hostname)) { - console.log('Direct media request by domain'); - flags.direct = true; - } else if (Constants.TEXT_ONLY_DOMAINS.includes(url.hostname)) { - console.log('Text-only embed request'); - flags.textOnly = true; - } else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) { - console.log('Forced instant view request'); - flags.forceInstantView = true; - } else if (prefix === 'dl' || prefix === 'dir') { - console.log('Direct media request by path prefix'); - flags.direct = true; - } - - /* The pxtwitter.com domain is deprecated and Tweets posted after deprecation - date will have a notice saying we've moved to fxtwitter.com! */ - if ( - Constants.DEPRECATED_DOMAIN_LIST.includes(url.hostname) && - BigInt(id?.match(/\d{2,20}/g)?.[0] || 0) > Constants.DEPRECATED_DOMAIN_EPOCH - ) { - console.log('Request to deprecated domain'); - flags.deprecated = true; - } - - /* TODO: Figure out what we're doing with FixTweet / FixupX branding in future */ - if (/fixup/g.test(url.href)) { - console.log(`We're using x domain`); - flags.isXDomain = true; - } else { - console.log(`We're using twitter domain`); - } - - const baseUrl = getBaseRedirectUrl(request); - - /* Check if request is to api.fxtwitter.com, or the tweet is appended with .json - Note that unlike TwitFix, FixTweet will never generate embeds for .json, and - in fact we only support .json because it's what people using TwitFix API would - be used to. */ - if ( - url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null || - Constants.API_HOST_LIST.includes(url.hostname) - ) { - console.log('JSON API request'); - flags.api = true; - } - - /* Direct media or API access bypasses bot check, returning same response regardless of UA */ - if (isBotUA || flags.direct || flags.api) { - if (isBotUA) { - console.log(`Matched bot UA ${userAgent}`); - } else { - console.log('Bypass bot check'); + const baseHostName = url.hostname.split('.').slice(-2).join('.'); + let realm = 'twitter'; + /* Override if in API_HOST_LIST. Note that we have to check full hostname for this. */ + if (Constants.API_HOST_LIST.includes(url.hostname)) { + realm = 'api'; + } else if (Constants.STANDARD_DOMAIN_LIST.includes(baseHostName)) { + realm = 'twitter'; } + /* Defaults to Twitter realm if unknown domain specified (such as the *.workers.dev hostname or deprecated domain) */ - /* This throws the necessary data to handleStatus (in status.ts) */ - const statusResponse = await handleStatus( - id?.match(/\d{2,20}/)?.[0] || '0', - mediaNumber ? parseInt(mediaNumber) : undefined, - userAgent, - flags, - language, - event, - request - ); - - /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */ - if (statusResponse.response) { - console.log('handleStatus sent response'); - return statusResponse.response; - } else if (statusResponse.text) { - console.log('handleStatus sent embed'); - /* We're checking if the User Agent is a bot again specifically in case they requested - direct media (d.fxtwitter.com, .mp4/.jpg, etc) but the Tweet contains no media. - - Since we obviously have no media to give the user, we'll just redirect to the Tweet. - Embeds will return as usual to bots as if direct media was never specified. */ - if (!isBotUA && !flags.api) { - const baseUrl = getBaseRedirectUrl(request); - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle || 'i'}/status/${id}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); - } - - let headers = Constants.RESPONSE_HEADERS; - - if (statusResponse.cacheControl) { - headers = { - ...headers, - 'cache-control': - baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : statusResponse.cacheControl - }; - } - - /* Return the response containing embed information */ - return new Response(statusResponse.text, { - headers: headers, - status: 200 - }); - } else { - /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ - return new Response(Strings.ERROR_UNKNOWN, { - headers: Constants.RESPONSE_HEADERS, - status: 500 - }); - } - } else { - globalThis.fetchCompletedTime = performance.now(); - /* A human has clicked a fxtwitter.com/:screen_name/status/:id link! - Obviously we just need to redirect to the Tweet directly.*/ - console.log('Matched human UA', userAgent); - - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); + console.log(`/${realm}${url.pathname}`); + return `/${realm}${url.pathname}`; } -}; +}); -/* Handler for User Profiles */ -const profileRequest = async (request: IRequest, event: FetchEvent, flags: InputFlags = {}) => { - const { handle } = request.params; - const url = new URL(request.url); - const userAgent = request.headers.get('User-Agent') || ''; - - /* User Agent matching for embed generators, bots, crawlers, and other automated - tools. It's pretty all-encompassing. Note that Firefox/92 is in here because - Discord sometimes uses the following UA: - - Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 - - I'm not sure why that specific one, it's pretty weird, but this edge case ensures - stuff keeps working. - - On the very rare off chance someone happens to be using specifically Firefox 92, - the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ - const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null; - - /* If not a valid screen name, we redirect to project GitHub */ - if (handle.match(/\w{1,15}/gi)?.[0] !== handle) { - return Response.redirect(Constants.REDIRECT_URL, 302); - } - const username = handle.match(/\w{1,15}/gi)?.[0] as string; - /* Check if request is to api.fxtwitter.com */ - if (Constants.API_HOST_LIST.includes(url.hostname)) { - console.log('JSON API request'); - flags.api = true; - } - - /* Direct media or API access bypasses bot check, returning same response regardless of UA */ - if (isBotUA || flags.api) { - if (isBotUA) { - console.log(`Matched bot UA ${userAgent}`); - } else { - console.log('Bypass bot check'); - } - - /* This throws the necessary data to handleStatus (in status.ts) */ - const profileResponse = await handleProfile(username, userAgent, flags, event); - - /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */ - if (profileResponse.response) { - console.log('handleProfile sent response'); - return profileResponse.response; - } else if (profileResponse.text) { - console.log('handleProfile sent embed'); - /* TODO This check has purpose in the original handleStatus handler, but I'm not sure if this edge case can happen here */ - const baseUrl = getBaseRedirectUrl(request); - /* Check for custom redirect */ - - if (!isBotUA) { - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); - } - - let headers = Constants.RESPONSE_HEADERS; - - if (profileResponse.cacheControl) { - headers = { - ...headers, - 'cache-control': - baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : profileResponse.cacheControl - }; - } - - /* Return the response containing embed information */ - return new Response(profileResponse.text, { - headers: headers, - status: 200 - }); - } else { - /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ - return new Response(Strings.ERROR_UNKNOWN, { - headers: { ...Constants.RESPONSE_HEADERS, 'cache-control': 'max-age=0' }, - status: 500 - }); - } - } else { - /* A human has clicked a fxtwitter.com/:screen_name link! - Obviously we just need to redirect to the user directly.*/ - console.log('Matched human UA', userAgent); - - const baseUrl = getBaseRedirectUrl(request); - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); - } -}; - -const genericTwitterRedirect = async (request: IRequest) => { - const url = new URL(request.url); - const baseUrl = getBaseRedirectUrl(request); - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}${url.pathname}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); -}; - -const versionRequest = async (request: IRequest) => { - globalThis.fetchCompletedTime = performance.now(); - return new Response( - Strings.VERSION_HTML.format({ - rtt: request.cf?.clientTcpRtt ? `🏓 ${request.cf.clientTcpRtt} ms RTT` : '', - colo: (request.cf?.colo as string) ?? '??', - httpversion: (request.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', - tlsversion: (request.cf?.tlsVersion as string) ?? 'Unknown TLS Version', - ip: - request.headers.get('x-real-ip') ?? request.headers.get('cf-connecting-ip') ?? 'Unknown IP', - city: (request.cf?.city as string) ?? 'Unknown City', - region: (request.cf?.region as string) ?? request.cf?.country ?? 'Unknown Region', - country: (request.cf?.country as string) ?? 'Unknown Country', - asn: `AS${request.cf?.asn ?? '??'} (${request.cf?.asOrganization ?? 'Unknown ASN'})`, - ua: sanitizeText(request.headers.get('user-agent') ?? 'Unknown User Agent') - }), - { - headers: { - ...Constants.RESPONSE_HEADERS, - 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate' - }, - status: 200 - } - ); -}; - -const setRedirectRequest = async (request: IRequest) => { - /* Query params */ - const { searchParams } = new URL(request.url); - let url = searchParams.get('url'); - - /* Check that origin either does not exist or is in our domain list */ - const origin = request.headers.get('origin'); - if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) { - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` - }), - { - headers: Constants.RESPONSE_HEADERS, - status: 403 - } - ); - } - - if (!url) { - /* Remove redirect URL */ - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` - }), - { - headers: { - 'set-cookie': `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`, - 'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`, - ...Constants.RESPONSE_HEADERS - }, - status: 200 - } - ); - } - - try { - new URL(url); - } catch (e) { - try { - new URL(`https://${url}`); - } catch (e) { - /* URL is not well-formed, remove */ - console.log('Invalid base redirect URL, removing cookie before redirect'); - - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` - }), - { - headers: { - 'set-cookie': `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`, - 'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join( - ' ' - )};`, - ...Constants.RESPONSE_HEADERS - }, - status: 200 - } - ); - } - - url = `https://${url}`; - } - - /* Set cookie for url */ - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Successfully set base redirect, you will now be redirected to ${sanitizeText( - url - )} rather than ${Constants.TWITTER_ROOT}` - }), - { - headers: { - 'set-cookie': `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`, - 'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`, - ...Constants.RESPONSE_HEADERS - }, - status: 200 - } - ); -}; - -/* TODO: is there any way to consolidate these stupid routes for itty-router? - I couldn't find documentation allowing for regex matching */ -router.get('/:prefix?/:handle/status/:id', statusRequest); -router.get('/:prefix?/:handle/status/:id/photo/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/photos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/video/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/videos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/photo/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/photos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/video/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/videos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/:language', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/:language', statusRequest); -router.get('/status/:id', statusRequest); -router.get('/status/:id/:language', statusRequest); -router.get('/version', versionRequest); -router.get('/set_base_redirect', setRedirectRequest); -// router.get('/v2/twitter/thread/:id', threadAPIProvider) - -/* Oembeds (used by Discord to enhance responses) - -Yes, I actually made the endpoint /owoembed. Deal with it. */ -router.get('/owoembed', async (request: IRequest) => { - globalThis.fetchCompletedTime = performance.now(); - console.log('oembed hit!'); - const { searchParams } = new URL(request.url); - - /* Fallbacks */ - const text = searchParams.get('text') || 'Twitter'; - const author = searchParams.get('author') || 'jack'; - const status = searchParams.get('status') || '20'; - - const random = Math.floor(Math.random() * Object.keys(motd).length); - const [name, url] = Object.entries(motd)[random]; - - const test = { - author_name: text, - author_url: `${Constants.TWITTER_ROOT}/${encodeURIComponent(author)}/status/${status}`, - /* Change provider name if tweet is on deprecated domain. */ - provider_name: - searchParams.get('deprecated') === 'true' ? Strings.DEPRECATED_DOMAIN_NOTICE_DISCORD : name, - provider_url: url, - title: Strings.DEFAULT_AUTHOR_TEXT, - type: 'link', - version: '1.0' - }; - /* Stringify and send it on its way! */ - return new Response(JSON.stringify(test), { - headers: { - ...Constants.RESPONSE_HEADERS, - 'content-type': 'application/json' +app.use( + '*', + sentry({ + dsn: SENTRY_DSN, + requestDataOptions: { + allowedHeaders: /(.*)/, + allowedSearchParams: /(.*)/ }, - status: 200 - }); + + integrations: [new RewriteFrames({ root: '/' })], + release: RELEASE_NAME + }) +); + +app.use('*', async (c, next) => { + /* Apply all headers from Constants.RESPONSE_HEADERS */ + for (const [header, value] of Object.entries(Constants.RESPONSE_HEADERS)) { + c.header(header, value); + } + await next(); }); -/* Pass through profile requests to Twitter. - We don't currently have custom profile cards yet, - but it's something we might do. Maybe. */ -router.get('/:handle', profileRequest); -router.get('/:handle/', profileRequest); -router.get('/i/events/:id', genericTwitterRedirect); -router.get('/hashtag/:hashtag', genericTwitterRedirect); +app.onError((err, c) => { + c.get('sentry').captureException(err); + /* workaround for silly TypeScript things */ + const error = err as Error; + console.error(error.stack); + c.status(200); + c.header('cache-control', noCache); + c.header('content-type', 'text/html'); -/* If we don't understand the route structure at all, we'll - redirect to GitHub (normal domains) or API docs (api.fxtwitter.com) */ -router.get('*', async (request: IRequest) => { - const url = new URL(request.url); - if (Constants.API_HOST_LIST.includes(url.hostname)) { - return Response.redirect(Constants.API_DOCS_URL, 302); - } - return Response.redirect(Constants.REDIRECT_URL, 302); + return c.text(Strings.ERROR_HTML); }); -/* Wrapper to handle caching, and misc things like catching robots.txt */ -export const cacheWrapper = async (request: Request, event?: FetchEvent): Promise => { - const startTime = performance.now(); - const userAgent = request.headers.get('User-Agent') || ''; - // https://developers.cloudflare.com/workers/examples/cache-api/ - const cacheUrl = new URL( - userAgent.includes('Telegram') - ? `${request.url}&telegram` - : userAgent.includes('Discord') - ? `${request.url}&discord` - : request.url - ); +app.use('*', logger()); - console.log(`Hello from ⛅ ${request.cf?.colo || 'UNK'}`); - console.log('userAgent', userAgent); - console.log('cacheUrl', cacheUrl); +app.use('*', async (c, next) => { + console.log(`Hello from ⛅ ${c.req.raw.cf?.colo || 'UNK'}`); + console.log('userAgent', c.req.header('user-agent')); + await next(); +}); - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; +app.use('*', cacheMiddleware()); +app.use('*', timing({ enabled: false })); - /* Itty-router doesn't seem to like routing file names because whatever, - so we just handle it in the caching layer instead. Kinda hacky, but whatever. */ - if (cacheUrl.pathname === '/robots.txt') { - return new Response(Strings.ROBOTS_TXT, { - headers: { - ...Constants.RESPONSE_HEADERS, - 'content-type': 'text/plain' - }, - status: 200 - }); +app.route(`/api`, api); +app.route(`/twitter`, twitter); + +app.all( + '/error', + async (c) => { + c.header('cache-control', noCache); + c.status(400); + return c.body('') } +); - switch (request.method) { - case 'GET': - if ( - !Constants.API_HOST_LIST.includes(cacheUrl.hostname) && - !request.headers?.get('Cookie')?.includes('base_redirect') - ) { - /* cache may be undefined in tests */ - const cachedResponse = await cache.match(cacheKey); - - if (cachedResponse) { - console.log('Cache hit'); - return cachedResponse; - } - - console.log('Cache miss'); - } - - const response = await router.handle(request, event); - - /* Store the fetched response as cacheKey - Use waitUntil so you can return the response without blocking on - writing to cache */ - try { - event && event.waitUntil(cache.put(cacheKey, response.clone())); - } catch (error) { - console.error((error as Error).stack); - } - - const endTime = performance.now(); - const timeSinceFetch = endTime - (globalThis.fetchCompletedTime || 0); - const timeSinceStart = endTime - startTime; - console.log( - `Request took ${timeSinceStart}ms, of which ${timeSinceFetch}ms was CPU time after last fetch` - ); - - return response; - /* Telegram sends this from Webpage Bot, and Cloudflare sends it if we purge cache, and we respect it. - PURGE is not defined in an RFC, but other servers like Nginx apparently use it. */ - case 'PURGE': - console.log('Purging cache as requested'); - await cache.delete(cacheKey); - return new Response('', { status: 200 }); - /* yes, we do give HEAD */ - case 'HEAD': - return new Response('', { - headers: Constants.RESPONSE_HEADERS, +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext) { + try { + return await app.fetch(request, env, ctx); + } catch (err) { + console.error(err); + console.log('Ouch, that error hurt so much Sentry couldnt catch it'); + + return new Response(Strings.ERROR_HTML, { + headers: { + ...Constants.RESPONSE_HEADERS, + 'content-type': 'text/html', + 'cache-control': noCache + }, status: 200 }); - /* We properly state our OPTIONS when asked */ - case 'OPTIONS': - return new Response('', { - headers: { - allow: Constants.RESPONSE_HEADERS.allow - }, - status: 204 - }); - default: - return new Response('', { status: 405 }); + } } }; - -/* Wrapper around Sentry, used for catching uncaught exceptions */ -const sentryWrapper = async (event: FetchEvent, test = false): Promise => { - let sentry: null | Toucan = null; - - if (typeof SENTRY_DSN !== 'undefined' && SENTRY_DSN && !test) { - /* We use Toucan for Sentry. Toucan is a Sentry SDK designed for Cloudflare Workers / DOs */ - sentry = new Toucan({ - dsn: SENTRY_DSN, - context: event, - request: event.request, - requestDataOptions: { - allowedHeaders: /(.*)/, - allowedSearchParams: /(.*)/ - }, - - /* TODO: Figure out what changed between @sentry/integration 7.65.0 and 7.66.0 - https://github.com/getsentry/sentry-javascript/compare/7.65.0...7.66.0 - which caused types to go apeshit */ - integrations: [new RewriteFrames({ root: '/' })], - /* event includes 'waitUntil', which is essential for Sentry logs to be delivered. - Also includes 'request' -- no need to set it separately. */ - release: RELEASE_NAME - }); - } - - /* Responds with either a returned response (good!!!) or returns - a crash response (bad!!!) */ - event.respondWith( - (async (): Promise => { - try { - return await cacheWrapper(event.request, event); - } catch (err: unknown) { - sentry && sentry.captureException(err); - - /* workaround for silly TypeScript things */ - const error = err as Error; - console.error(error.stack); - - return new Response(Strings.ERROR_HTML, { - headers: { - ...Constants.RESPONSE_HEADERS, - 'content-type': 'text/html', - 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate' - }, - status: 200 - }); - } - })() - ); -}; - -/* Event to receive web requests on Cloudflare Worker */ -addEventListener('fetch', (event: FetchEvent) => { - sentryWrapper(event); -}); From d06762ffe9300961875df42073a00b5dd6468ea6 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 9 Nov 2023 04:45:47 -0500 Subject: [PATCH 04/30] Bugfixes --- src/constants.ts | 1 - src/embed/status.ts | 4 +- src/providers/twitter/conversation.ts | 2 +- src/realms/api/router.ts | 17 ++------ src/realms/common/version.ts | 39 +++++++----------- src/realms/twitter/router.ts | 56 ++++++++++++-------------- src/realms/twitter/routes/redirects.ts | 8 ++-- src/realms/twitter/routes/status.ts | 6 +-- src/user.ts | 6 +-- src/worker.ts | 10 ++--- 10 files changed, 58 insertions(+), 91 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index dada8b7..0c2c911 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -61,7 +61,6 @@ export const Constants = { 'content-type': 'text/html;charset=UTF-8', 'x-powered-by': `${RELEASE_NAME}`, 'x-trans-rights': 'true', - 'cache-control': 'max-age=3600', // Can be overriden in some cases, like unfinished poll tweets 'Vary': 'Accept-Encoding, User-Agent' }, API_RESPONSE_HEADERS: { diff --git a/src/embed/status.ts b/src/embed/status.ts index 8340bbc..10cfcb1 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -77,7 +77,7 @@ export const handleStatus = async ( for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { c.header(header, value); } - return c.text(JSON.stringify(api)); + return c.json(api); } if (tweet === null) { @@ -423,7 +423,7 @@ export const handleStatus = async ( const lang = tweet.lang === null ? 'en' : tweet.lang || 'en'; /* Finally, after all that work we return the response HTML! */ - return c.text(Strings.BASE_HTML.format({ + return c.html(Strings.BASE_HTML.format({ lang: `lang="${lang}"`, headers: headers.join(''), body: ivbody diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 4e1473f..4e8cdf3 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -519,5 +519,5 @@ export const threadAPIProvider = async (c: Context) => { for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { c.header(header, value); } - return c.text(JSON.stringify(processedResponse)); + return c.json(processedResponse); }; diff --git a/src/realms/api/router.ts b/src/realms/api/router.ts index 365ce6a..22d2e2b 100644 --- a/src/realms/api/router.ts +++ b/src/realms/api/router.ts @@ -6,19 +6,8 @@ import { Strings } from '../../strings'; export const api = new Hono(); /* Current v1 API endpoints. Currently, these still go through the Twitter embed requests. API v2+ won't do this. */ -api.get('/:handle?/status/:id/:language?', statusRequest); -api.get( - '/:handle?/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', - statusRequest -); +api.get('/status/:id/:language?', statusRequest); +api.get('/:handle/status/:id/:language?', statusRequest); api.get('/:handle', profileRequest); - -api.get( - '/robots.txt', - async (c) => { - c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); - c.status(200); - return c.text(Strings.ROBOTS_TXT); - } -); +api.get('/robots.txt', async (c) => c.text(Strings.ROBOTS_TXT)); diff --git a/src/realms/common/version.ts b/src/realms/common/version.ts index 0555431..4ecae97 100644 --- a/src/realms/common/version.ts +++ b/src/realms/common/version.ts @@ -1,29 +1,20 @@ import { Context } from 'hono'; -import { Constants } from '../../constants'; import { sanitizeText } from '../../helpers/utils'; import { Strings } from '../../strings'; -export const versionRoute = async (context: Context) => { - const request = context.req; - return new Response( - Strings.VERSION_HTML.format({ - rtt: request.raw.cf?.clientTcpRtt ? `🏓 ${request.raw.cf.clientTcpRtt} ms RTT` : '', - colo: (request.raw.cf?.colo as string) ?? '??', - httpversion: (request.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', - tlsversion: (request.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version', - ip: request.header('x-real-ip') ?? request.header('cf-connecting-ip') ?? 'Unknown IP', - city: (request.raw.cf?.city as string) ?? 'Unknown City', - region: (request.raw.cf?.region as string) ?? request.raw.cf?.country ?? 'Unknown Region', - country: (request.raw.cf?.country as string) ?? 'Unknown Country', - asn: `AS${request.raw.cf?.asn ?? '??'} (${request.raw.cf?.asOrganization ?? 'Unknown ASN'})`, - ua: sanitizeText(request.header('user-agent') ?? 'Unknown User Agent') - }), - { - headers: { - ...Constants.RESPONSE_HEADERS, - 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate' - }, - status: 200 - } - ); +export const versionRoute = async (c: Context) => { + c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); + const req = c.req; + return c.html(Strings.VERSION_HTML.format({ + rtt: req.raw.cf?.clientTcpRtt ? `🏓 ${req.raw.cf.clientTcpRtt} ms RTT` : '', + colo: (req.raw.cf?.colo as string) ?? '??', + httpversion: (req.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', + tlsversion: (req.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version', + ip: req.header('x-real-ip') ?? req.header('cf-connecting-ip') ?? 'Unknown IP', + city: (req.raw.cf?.city as string) ?? 'Unknown City', + region: (req.raw.cf?.region as string) ?? req.raw.cf?.country ?? 'Unknown Region', + country: (req.raw.cf?.country as string) ?? 'Unknown Country', + asn: `AS${req.raw.cf?.asn ?? '??'} (${req.raw.cf?.asOrganization ?? 'Unknown ASN'})`, + ua: sanitizeText(req.header('user-agent') ?? 'Unknown User Agent') + })) }; diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts index c19ad00..28bc83d 100644 --- a/src/realms/twitter/router.ts +++ b/src/realms/twitter/router.ts @@ -10,37 +10,6 @@ import { oembed } from './routes/oembed'; export const twitter = new Hono(); -twitter.get('/status/:id', statusRequest); -// twitter.get('/:handle/status/:id', statusRequest); -// twitter.get('/:prefix/:handle/status/:id/:language?', statusRequest); -// twitter.get( -// '/:prefix/:handle/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', -// statusRequest -// ); -// twitter.get('/:handle?/:endpoint{status(es)?}/:id/:language?', statusRequest); -// twitter.get( -// '/:handle?/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', -// statusRequest -// ); - -twitter.get('/version', versionRoute); -twitter.get('/set_base_redirect', setRedirectRequest); -twitter.get('/oembed', oembed); - -twitter.get( - '/robots.txt', - async (c) => { - c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); - c.status(200); - return c.text(Strings.ROBOTS_TXT); - } -); - -twitter.get('/i/events/:id', genericTwitterRedirect); -twitter.get('/hashtag/:hashtag', genericTwitterRedirect); - -twitter.get('/:handle', profileRequest); - export const getBaseRedirectUrl = (c: Context) => { const baseRedirect = c.req.header('cookie')?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0]; @@ -56,3 +25,28 @@ export const getBaseRedirectUrl = (c: Context) => { return Constants.TWITTER_ROOT; }; + + +twitter.get('/status/:id', statusRequest); +twitter.get('/:handle/status/:id', statusRequest); +twitter.get('/:prefix/:handle/status/:id/:language?', statusRequest); +twitter.get( + '/:prefix/:handle/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + statusRequest +); +twitter.get('/:handle?/:endpoint{status(es)?}/:id/:language?', statusRequest); +twitter.get( + '/:handle?/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + statusRequest +); + +twitter.get('/version', versionRoute); +twitter.get('/set_base_redirect', setRedirectRequest); +twitter.get('/oembed', oembed); + +twitter.get('/robots.txt', async (c) => c.text(Strings.ROBOTS_TXT)); + +twitter.get('/i/events/:id', genericTwitterRedirect); +twitter.get('/hashtag/:hashtag', genericTwitterRedirect); + +twitter.get('/:handle', profileRequest); \ No newline at end of file diff --git a/src/realms/twitter/routes/redirects.ts b/src/realms/twitter/routes/redirects.ts index f026d01..51b208a 100644 --- a/src/realms/twitter/routes/redirects.ts +++ b/src/realms/twitter/routes/redirects.ts @@ -27,7 +27,7 @@ export const setRedirectRequest = async (c: Context) => { if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) { c.status(403); - return c.text(Strings.MESSAGE_HTML.format({ + return c.html(Strings.MESSAGE_HTML.format({ message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` })) } @@ -40,7 +40,7 @@ export const setRedirectRequest = async (c: Context) => { // eslint-disable-next-line sonarjs/no-duplicate-string c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); c.status(200); - return c.text(Strings.MESSAGE_HTML.format({ + return c.html(Strings.MESSAGE_HTML.format({ message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` })) } @@ -57,7 +57,7 @@ export const setRedirectRequest = async (c: Context) => { c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`); c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); c.status(200); - return c.text(Strings.MESSAGE_HTML.format({ + return c.html(Strings.MESSAGE_HTML.format({ message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` })) } @@ -68,7 +68,7 @@ export const setRedirectRequest = async (c: Context) => { c.header('set-cookie', `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`); c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); - return c.text(Strings.MESSAGE_HTML.format({ + return c.html(Strings.MESSAGE_HTML.format({ message: `Successfully set base redirect, you will now be redirected to ${sanitizeText(url)} rather than ${Constants.TWITTER_ROOT}` })) }; diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts index 50e07e2..64d84aa 100644 --- a/src/realms/twitter/routes/status.ts +++ b/src/realms/twitter/routes/status.ts @@ -6,11 +6,7 @@ import { Strings } from '../../../strings'; /* Handler for status (Tweet) request */ export const statusRequest = async (c: Context) => { - const handle = c.req.param('handle'); - const id = c.req.param('id'); - const mediaNumber = c.req.param('mediaNumber'); - const language = c.req.param('language'); - const prefix = c.req.param('prefix'); + const { handle, id, mediaNumber, language, prefix } = c.req.param(); const url = new URL(c.req.url); const flags: InputFlags = {}; diff --git a/src/user.ts b/src/user.ts index 2ea3dda..005d977 100644 --- a/src/user.ts +++ b/src/user.ts @@ -4,7 +4,7 @@ import { Strings } from './strings'; import { userAPI } from './providers/twitter/profile'; export const returnError = (c: Context, error: string): Response => { - return c.text(Strings.BASE_HTML.format({ + return c.html(Strings.BASE_HTML.format({ lang: '', headers: [ ``, @@ -32,7 +32,7 @@ export const handleProfile = async ( for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { c.header(header, value); } - return c.text(JSON.stringify(api)); + return c.json(api); } /* If there was any errors fetching the User, we'll return it */ @@ -51,7 +51,7 @@ export const handleProfile = async ( // TODO Add card creation logic here /* Finally, after all that work we return the response HTML! */ - return c.text(Strings.BASE_HTML.format({ + return c.html(Strings.BASE_HTML.format({ lang: `lang="en"`, headers: headers.join('') })); diff --git a/src/worker.ts b/src/worker.ts index d4f7e76..b8d3a75 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -64,13 +64,11 @@ app.onError((err, c) => { console.error(error.stack); c.status(200); c.header('cache-control', noCache); - c.header('content-type', 'text/html'); - - return c.text(Strings.ERROR_HTML); + return c.html(Strings.ERROR_HTML); }); -app.use('*', logger()); +// app.use('*', logger()); app.use('*', async (c, next) => { console.log(`Hello from ⛅ ${c.req.raw.cf?.colo || 'UNK'}`); @@ -81,8 +79,8 @@ app.use('*', async (c, next) => { app.use('*', cacheMiddleware()); app.use('*', timing({ enabled: false })); -app.route(`/api`, api); app.route(`/twitter`, twitter); +app.route(`/api`, api); app.all( '/error', @@ -104,7 +102,7 @@ export default { return new Response(Strings.ERROR_HTML, { headers: { ...Constants.RESPONSE_HEADERS, - 'content-type': 'text/html', + 'content-type': 'text/html;charset=utf-8', 'cache-control': noCache }, status: 200 From 66dd15ea6370115b1b4e19b78a9be97685f42e4b Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 9 Nov 2023 04:47:26 -0500 Subject: [PATCH 05/30] Switch these back --- src/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.ts b/src/worker.ts index b8d3a75..59130c4 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -79,8 +79,8 @@ app.use('*', async (c, next) => { app.use('*', cacheMiddleware()); app.use('*', timing({ enabled: false })); -app.route(`/twitter`, twitter); app.route(`/api`, api); +app.route(`/twitter`, twitter); app.all( '/error', From e4b9efd7938220a79a96f5dc12678ff6a1088f32 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 00:57:07 -0500 Subject: [PATCH 06/30] Limit length to 255 not 256 (fixes #474) --- src/embed/status.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/embed/status.ts b/src/embed/status.ts index 10cfcb1..c39e358 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -410,7 +410,7 @@ export const handleStatus = async ( ``.format( { base: Constants.HOST_URL, - text: encodeURIComponent(truncateWithEllipsis(authorText, 256)), + text: encodeURIComponent(truncateWithEllipsis(authorText, 255)), deprecatedFlag: flags?.deprecated ? '&deprecated=true' : '', status: encodeURIComponent(status), author: encodeURIComponent(tweet.author?.screen_name || ''), From 85eb2cead117140f87e1de78b11526a81ee87ec1 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 01:00:25 -0500 Subject: [PATCH 07/30] Fix routing and logging --- src/realms/api/router.ts | 4 ++++ src/realms/twitter/router.ts | 20 +++++++++++--------- src/worker.ts | 15 +++++++++++---- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/realms/api/router.ts b/src/realms/api/router.ts index 22d2e2b..ce50151 100644 --- a/src/realms/api/router.ts +++ b/src/realms/api/router.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import { statusRequest } from '../twitter/routes/status'; import { profileRequest } from '../twitter/routes/profile'; import { Strings } from '../../strings'; +import { Constants } from '../../constants'; export const api = new Hono(); @@ -11,3 +12,6 @@ api.get('/:handle/status/:id/:language?', statusRequest); api.get('/:handle', profileRequest); api.get('/robots.txt', async (c) => c.text(Strings.ROBOTS_TXT)); + + +api.all('*', async (c) => c.redirect(Constants.API_DOCS_URL, 302)); \ No newline at end of file diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts index 28bc83d..9c1af55 100644 --- a/src/realms/twitter/router.ts +++ b/src/realms/twitter/router.ts @@ -26,18 +26,18 @@ export const getBaseRedirectUrl = (c: Context) => { return Constants.TWITTER_ROOT; }; +/* Workaround for some dumb maybe-build time issue where statusRequest isn't ready or something because none of these trigger*/ +const tweetRequest = async (c: Context) => await statusRequest(c); -twitter.get('/status/:id', statusRequest); -twitter.get('/:handle/status/:id', statusRequest); -twitter.get('/:prefix/:handle/status/:id/:language?', statusRequest); +twitter.get('/:prefix?/:handle?/:endpoint{status(es)?}/:id/:language?', tweetRequest); +twitter.get(':handle?/:endpoint{status(es)?}/:id/:language?', tweetRequest); twitter.get( - '/:prefix/:handle/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', - statusRequest + '/:prefix?/:handle/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + tweetRequest ); -twitter.get('/:handle?/:endpoint{status(es)?}/:id/:language?', statusRequest); twitter.get( - '/:handle?/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', - statusRequest + '/:handle/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + tweetRequest ); twitter.get('/version', versionRoute); @@ -49,4 +49,6 @@ twitter.get('/robots.txt', async (c) => c.text(Strings.ROBOTS_TXT)); twitter.get('/i/events/:id', genericTwitterRedirect); twitter.get('/hashtag/:hashtag', genericTwitterRedirect); -twitter.get('/:handle', profileRequest); \ No newline at end of file +twitter.get('/:handle', profileRequest); + +twitter.all('*', async (c) => c.redirect(Constants.REDIRECT_URL, 302)); \ No newline at end of file diff --git a/src/worker.ts b/src/worker.ts index 59130c4..9f48007 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -25,8 +25,13 @@ const app = new Hono<{ Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: Analy /* Override if in API_HOST_LIST. Note that we have to check full hostname for this. */ if (Constants.API_HOST_LIST.includes(url.hostname)) { realm = 'api'; + console.log('API realm'); } else if (Constants.STANDARD_DOMAIN_LIST.includes(baseHostName)) { + console.log() realm = 'twitter'; + console.log('Twitter realm'); + } else { + console.log(`Domain not assigned to realm, falling back to Twitter: ${url.hostname}`); } /* Defaults to Twitter realm if unknown domain specified (such as the *.workers.dev hostname or deprecated domain) */ @@ -59,16 +64,18 @@ app.use('*', async (c, next) => { app.onError((err, c) => { c.get('sentry').captureException(err); - /* workaround for silly TypeScript things */ - const error = err as Error; - console.error(error.stack); + console.error(err.stack); c.status(200); c.header('cache-control', noCache); return c.html(Strings.ERROR_HTML); }); -// app.use('*', logger()); + const customLogger = (message: string, ...rest: string[]) => { + console.log(message, ...rest); +}; + +app.use('*', logger(customLogger)); app.use('*', async (c, next) => { console.log(`Hello from ⛅ ${c.req.raw.cf?.colo || 'UNK'}`); From b5ee1b8fe7349281aba70cfafc00530581d2a217 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 01:00:37 -0500 Subject: [PATCH 08/30] Run prettier --- src/caches.ts | 12 ++--- src/embed/status.ts | 30 +++++++------ src/fetch.ts | 11 +++-- src/providers/twitter/conversation.ts | 6 ++- src/providers/twitter/processor.ts | 2 +- src/realms/api/router.ts | 5 +-- src/realms/common/version.ts | 26 ++++++----- src/realms/twitter/router.ts | 4 +- src/realms/twitter/routes/oembed.ts | 4 +- src/realms/twitter/routes/profile.ts | 4 +- src/realms/twitter/routes/redirects.ts | 62 ++++++++++++++++++-------- src/realms/twitter/routes/status.ts | 5 ++- src/user.ts | 28 +++++++----- src/worker.ts | 23 +++++----- 14 files changed, 131 insertions(+), 91 deletions(-) diff --git a/src/caches.ts b/src/caches.ts index 2b2fb6a..48c3938 100644 --- a/src/caches.ts +++ b/src/caches.ts @@ -63,17 +63,17 @@ export const cacheMiddleware = (): MiddlewareHandler => async (c, next) => { case 'DELETE': console.log('Purging cache as requested'); await cache.delete(cacheKey); - return c.text('') + return c.text(''); /* yes, we do give HEAD */ case 'HEAD': - return c.text('') + return c.text(''); /* We properly state our OPTIONS when asked */ case 'OPTIONS': - c.header('allow', Constants.RESPONSE_HEADERS.allow) - c.status(204) + c.header('allow', Constants.RESPONSE_HEADERS.allow); + c.status(204); return c.text(''); default: c.status(405); - return c.text('') + return c.text(''); } -}; \ No newline at end of file +}; diff --git a/src/embed/status.ts b/src/embed/status.ts index c39e358..e4f39d3 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -10,13 +10,15 @@ import { constructTwitterThread } from '../providers/twitter/conversation'; import { Context } from 'hono'; export const returnError = (c: Context, error: string): Response => { - return c.text(Strings.BASE_HTML.format({ - lang: '', - headers: [ - ``, - `` - ].join('') - })); + return c.text( + Strings.BASE_HTML.format({ + lang: '', + headers: [ + ``, + `` + ].join('') + }) + ); }; /* Handler for Twitter statuses (Tweets). Like Twitter, we use the terminologies interchangably. */ @@ -26,7 +28,7 @@ export const handleStatus = async ( mediaNumber: number | undefined, userAgent: string, flags: InputFlags, - language: string, + language: string // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { console.log('Direct?', flags?.direct); @@ -423,9 +425,11 @@ export const handleStatus = async ( const lang = tweet.lang === null ? 'en' : tweet.lang || 'en'; /* Finally, after all that work we return the response HTML! */ - return c.html(Strings.BASE_HTML.format({ - lang: `lang="${lang}"`, - headers: headers.join(''), - body: ivbody - }).replace(/>(\s+)<')); + return c.html( + Strings.BASE_HTML.format({ + lang: `lang="${lang}"`, + headers: headers.join(''), + body: ivbody + }).replace(/>(\s+)<') + ); }; diff --git a/src/fetch.ts b/src/fetch.ts index 1ab32d8..760f3fb 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -172,7 +172,9 @@ export const twitterFetch = async ( } !useElongator && c.executionCtx && - c.executionCtx.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); + c.executionCtx.waitUntil( + cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) + ); if (useElongator) { console.log('Elongator request failed, trying again without it'); wasElongatorDisabled = true; @@ -202,7 +204,9 @@ export const twitterFetch = async ( if (!useElongator && remainingRateLimit < 10) { console.log(`Purging token on this edge due to low rate limit remaining`); c.executionCtx && - c.executionCtx.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); + c.executionCtx.waitUntil( + cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) + ); } if (!validateFunction(response)) { @@ -250,7 +254,8 @@ export const fetchUser = async ( typeof c.env.TwitterProxy !== 'undefined' ) ): Promise => { - return (await twitterFetch(c, + return (await twitterFetch( + c, `${ Constants.TWITTER_ROOT }/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?variables=${encodeURIComponent( diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 4e8cdf3..5ee022f 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -11,7 +11,8 @@ export const fetchTweetDetail = async ( useElongator = typeof c.env.TwitterProxy !== 'undefined', cursor: string | null = null ): Promise => { - return (await twitterFetch(c, + return (await twitterFetch( + c, `${ Constants.TWITTER_ROOT }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( @@ -95,7 +96,8 @@ export const fetchByRestId = async ( typeof c.env.TwitterProxy !== 'undefined' ) ): Promise => { - return (await twitterFetch(c, + return (await twitterFetch( + c, `${ Constants.TWITTER_ROOT }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index be43668..58d140b 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -14,7 +14,7 @@ export const buildAPITweet = async ( tweet: GraphQLTweet, language: string | undefined, threadPiece = false, - legacyAPI = false, + legacyAPI = false // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { const apiTweet = {} as APITweet; diff --git a/src/realms/api/router.ts b/src/realms/api/router.ts index ce50151..f692ac8 100644 --- a/src/realms/api/router.ts +++ b/src/realms/api/router.ts @@ -11,7 +11,6 @@ api.get('/status/:id/:language?', statusRequest); api.get('/:handle/status/:id/:language?', statusRequest); api.get('/:handle', profileRequest); -api.get('/robots.txt', async (c) => c.text(Strings.ROBOTS_TXT)); +api.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT)); - -api.all('*', async (c) => c.redirect(Constants.API_DOCS_URL, 302)); \ No newline at end of file +api.all('*', async c => c.redirect(Constants.API_DOCS_URL, 302)); diff --git a/src/realms/common/version.ts b/src/realms/common/version.ts index 4ecae97..da8e0c1 100644 --- a/src/realms/common/version.ts +++ b/src/realms/common/version.ts @@ -5,16 +5,18 @@ import { Strings } from '../../strings'; export const versionRoute = async (c: Context) => { c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); const req = c.req; - return c.html(Strings.VERSION_HTML.format({ - rtt: req.raw.cf?.clientTcpRtt ? `🏓 ${req.raw.cf.clientTcpRtt} ms RTT` : '', - colo: (req.raw.cf?.colo as string) ?? '??', - httpversion: (req.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', - tlsversion: (req.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version', - ip: req.header('x-real-ip') ?? req.header('cf-connecting-ip') ?? 'Unknown IP', - city: (req.raw.cf?.city as string) ?? 'Unknown City', - region: (req.raw.cf?.region as string) ?? req.raw.cf?.country ?? 'Unknown Region', - country: (req.raw.cf?.country as string) ?? 'Unknown Country', - asn: `AS${req.raw.cf?.asn ?? '??'} (${req.raw.cf?.asOrganization ?? 'Unknown ASN'})`, - ua: sanitizeText(req.header('user-agent') ?? 'Unknown User Agent') - })) + return c.html( + Strings.VERSION_HTML.format({ + rtt: req.raw.cf?.clientTcpRtt ? `🏓 ${req.raw.cf.clientTcpRtt} ms RTT` : '', + colo: (req.raw.cf?.colo as string) ?? '??', + httpversion: (req.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', + tlsversion: (req.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version', + ip: req.header('x-real-ip') ?? req.header('cf-connecting-ip') ?? 'Unknown IP', + city: (req.raw.cf?.city as string) ?? 'Unknown City', + region: (req.raw.cf?.region as string) ?? req.raw.cf?.country ?? 'Unknown Region', + country: (req.raw.cf?.country as string) ?? 'Unknown Country', + asn: `AS${req.raw.cf?.asn ?? '??'} (${req.raw.cf?.asOrganization ?? 'Unknown ASN'})`, + ua: sanitizeText(req.header('user-agent') ?? 'Unknown User Agent') + }) + ); }; diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts index 9c1af55..748659e 100644 --- a/src/realms/twitter/router.ts +++ b/src/realms/twitter/router.ts @@ -44,11 +44,11 @@ twitter.get('/version', versionRoute); twitter.get('/set_base_redirect', setRedirectRequest); twitter.get('/oembed', oembed); -twitter.get('/robots.txt', async (c) => c.text(Strings.ROBOTS_TXT)); +twitter.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT)); twitter.get('/i/events/:id', genericTwitterRedirect); twitter.get('/hashtag/:hashtag', genericTwitterRedirect); twitter.get('/:handle', profileRequest); -twitter.all('*', async (c) => c.redirect(Constants.REDIRECT_URL, 302)); \ No newline at end of file +twitter.all('*', async c => c.redirect(Constants.REDIRECT_URL, 302)); diff --git a/src/realms/twitter/routes/oembed.ts b/src/realms/twitter/routes/oembed.ts index cd5698f..3285d42 100644 --- a/src/realms/twitter/routes/oembed.ts +++ b/src/realms/twitter/routes/oembed.ts @@ -27,8 +27,8 @@ export const oembed = async (c: Context) => { type: 'link', version: '1.0' }; - c.header('content-type', 'application/json') + c.header('content-type', 'application/json'); c.status(200); /* Stringify and send it on its way! */ - return c.text(JSON.stringify(test)) + return c.text(JSON.stringify(test)); }; diff --git a/src/realms/twitter/routes/profile.ts b/src/realms/twitter/routes/profile.ts index 78de44c..9394d85 100644 --- a/src/realms/twitter/routes/profile.ts +++ b/src/realms/twitter/routes/profile.ts @@ -38,7 +38,7 @@ export const profileRequest = async (c: Context) => { /* Do not cache if using a custom redirect */ const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - + if (cacheControl) { c.header('cache-control', cacheControl); } @@ -65,6 +65,6 @@ export const profileRequest = async (c: Context) => { Obviously we just need to redirect to the user directly.*/ console.log('Matched human UA', userAgent); - return c.redirect(`${baseUrl}/${handle}`, 302) + return c.redirect(`${baseUrl}/${handle}`, 302); } }; diff --git a/src/realms/twitter/routes/redirects.ts b/src/realms/twitter/routes/redirects.ts index 51b208a..f44d6f2 100644 --- a/src/realms/twitter/routes/redirects.ts +++ b/src/realms/twitter/routes/redirects.ts @@ -13,7 +13,7 @@ export const genericTwitterRedirect = async (c: Context) => { if (cacheControl) { c.header('cache-control', cacheControl); } - + return c.redirect(`${baseUrl}${url.pathname}`, 302); }; @@ -27,22 +27,31 @@ export const setRedirectRequest = async (c: Context) => { if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) { c.status(403); - return c.html(Strings.MESSAGE_HTML.format({ - message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` - })) + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` + }) + ); } if (!url) { - /* Remove redirect URL */ // eslint-disable-next-line sonarjs/no-duplicate-string - c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`); + c.header( + 'set-cookie', + `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly` + ); // eslint-disable-next-line sonarjs/no-duplicate-string - c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); + c.header( + 'content-security-policy', + `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` + ); c.status(200); - return c.html(Strings.MESSAGE_HTML.format({ - message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` - })) + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` + }) + ); } try { @@ -54,21 +63,36 @@ export const setRedirectRequest = async (c: Context) => { /* URL is not well-formed, remove */ console.log('Invalid base redirect URL, removing cookie before redirect'); - c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`); - c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); + c.header( + 'set-cookie', + `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly` + ); + c.header( + 'content-security-policy', + `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` + ); c.status(200); - return c.html(Strings.MESSAGE_HTML.format({ - message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` - })) + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` + }) + ); } url = `https://${url}`; } c.header('set-cookie', `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`); - c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`); + c.header( + 'content-security-policy', + `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` + ); - return c.html(Strings.MESSAGE_HTML.format({ - message: `Successfully set base redirect, you will now be redirected to ${sanitizeText(url)} rather than ${Constants.TWITTER_ROOT}` - })) + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Successfully set base redirect, you will now be redirected to ${sanitizeText( + url + )} rather than ${Constants.TWITTER_ROOT}` + }) + ); }; diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts index 64d84aa..625cc8d 100644 --- a/src/realms/twitter/routes/status.ts +++ b/src/realms/twitter/routes/status.ts @@ -101,7 +101,8 @@ export const statusRequest = async (c: Context) => { } /* This throws the necessary data to handleStatus (in status.ts) */ - const statusResponse = await handleStatus(c, + const statusResponse = await handleStatus( + c, id?.match(/\d{2,20}/)?.[0] || '0', mediaNumber ? parseInt(mediaNumber) : undefined, userAgent, @@ -124,7 +125,7 @@ export const statusRequest = async (c: Context) => { Embeds will return as usual to bots as if direct media was never specified. */ if (!isBotUA && !flags.api) { const baseUrl = getBaseRedirectUrl(c); - + return c.redirect(`${baseUrl}/${handle || 'i'}/status/${id}`, 302); } diff --git a/src/user.ts b/src/user.ts index 005d977..256ead3 100644 --- a/src/user.ts +++ b/src/user.ts @@ -4,20 +4,22 @@ import { Strings } from './strings'; import { userAPI } from './providers/twitter/profile'; export const returnError = (c: Context, error: string): Response => { - return c.html(Strings.BASE_HTML.format({ - lang: '', - headers: [ - ``, - `` - ].join('') - })); + return c.html( + Strings.BASE_HTML.format({ + lang: '', + headers: [ + ``, + `` + ].join('') + }) + ); }; /* Handler for Twitter users */ export const handleProfile = async ( c: Context, username: string, - flags: InputFlags, + flags: InputFlags ): Promise => { console.log('Direct?', flags?.direct); @@ -51,8 +53,10 @@ export const handleProfile = async ( // TODO Add card creation logic here /* Finally, after all that work we return the response HTML! */ - return c.html(Strings.BASE_HTML.format({ - lang: `lang="en"`, - headers: headers.join('') - })); + return c.html( + Strings.BASE_HTML.format({ + lang: `lang="en"`, + headers: headers.join('') + }) + ); }; diff --git a/src/worker.ts b/src/worker.ts index 9f48007..8de5029 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -11,7 +11,9 @@ import { cacheMiddleware } from './caches'; const noCache = 'max-age=0, no-cache, no-store, must-revalidate'; -const app = new Hono<{ Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: AnalyticsEngineDataset } }>({ +const app = new Hono<{ + Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: AnalyticsEngineDataset }; +}>({ getPath: req => { let url: URL; @@ -27,7 +29,7 @@ const app = new Hono<{ Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: Analy realm = 'api'; console.log('API realm'); } else if (Constants.STANDARD_DOMAIN_LIST.includes(baseHostName)) { - console.log() + console.log(); realm = 'twitter'; console.log('Twitter realm'); } else { @@ -71,7 +73,7 @@ app.onError((err, c) => { return c.html(Strings.ERROR_HTML); }); - const customLogger = (message: string, ...rest: string[]) => { +const customLogger = (message: string, ...rest: string[]) => { console.log(message, ...rest); }; @@ -89,14 +91,11 @@ app.use('*', timing({ enabled: false })); app.route(`/api`, api); app.route(`/twitter`, twitter); -app.all( - '/error', - async (c) => { - c.header('cache-control', noCache); - c.status(400); - return c.body('') - } -); +app.all('/error', async c => { + c.header('cache-control', noCache); + c.status(400); + return c.body(''); +}); export default { async fetch(request: Request, env: Env, ctx: ExecutionContext) { @@ -105,7 +104,7 @@ export default { } catch (err) { console.error(err); console.log('Ouch, that error hurt so much Sentry couldnt catch it'); - + return new Response(Strings.ERROR_HTML, { headers: { ...Constants.RESPONSE_HEADERS, From 822a744cc567dc8f8994215f6d52886d62cc36ea Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 01:04:20 -0500 Subject: [PATCH 09/30] Move these now that prettier was run --- src/realms/twitter/routes/redirects.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/realms/twitter/routes/redirects.ts b/src/realms/twitter/routes/redirects.ts index f44d6f2..cfff1f2 100644 --- a/src/realms/twitter/routes/redirects.ts +++ b/src/realms/twitter/routes/redirects.ts @@ -36,13 +36,13 @@ export const setRedirectRequest = async (c: Context) => { if (!url) { /* Remove redirect URL */ - // eslint-disable-next-line sonarjs/no-duplicate-string c.header( + // eslint-disable-next-line sonarjs/no-duplicate-string 'set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly` ); - // eslint-disable-next-line sonarjs/no-duplicate-string c.header( + // eslint-disable-next-line sonarjs/no-duplicate-string 'content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` ); From 7b4f56f90da5b5030ff4a4ec19845c4d4b1ecd75 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 03:30:06 -0500 Subject: [PATCH 10/30] Fix this after prettier --- src/helpers/graphql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index 310dd75..0e10623 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -16,9 +16,9 @@ export const isGraphQLTweet = (response: unknown): response is GraphQLTweet => { return ( typeof response === 'object' && response !== null && - // @ts-expect-error it's 6 am please let me sleep (('__typename' in response && (response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults')) || + // @ts-expect-error it's 6 am please let me sleep ('legacy' in response && response.legacy?.full_text)) ); }; From c9704331ad8919bf09887725a1d99484d90365b7 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 03:30:13 -0500 Subject: [PATCH 11/30] Bump fakeChromeVersion --- src/helpers/useragent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/useragent.ts b/src/helpers/useragent.ts index d09d536..42788f4 100644 --- a/src/helpers/useragent.ts +++ b/src/helpers/useragent.ts @@ -1,6 +1,6 @@ /* We keep this value up-to-date for making our requests to Twitter as indistinguishable from normal user traffic as possible. */ -const fakeChromeVersion = 118; +const fakeChromeVersion = 119; const platformWindows = 'Windows NT 10.0; Win64; x64'; const platformMac = 'Macintosh; Intel Mac OS X 10_15_7'; const platformLinux = 'X11; Linux x86_64'; From f27a5d4be84ac94728d501c02aced00faf56a8c1 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 03:30:37 -0500 Subject: [PATCH 12/30] Minimum work to get tests plugged in, still broken --- src/worker.ts | 8 +++++--- test/index.test.ts | 26 +++++++++++++------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/worker.ts b/src/worker.ts index 8de5029..c0455ab 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -11,7 +11,7 @@ import { cacheMiddleware } from './caches'; const noCache = 'max-age=0, no-cache, no-store, must-revalidate'; -const app = new Hono<{ +export const app = new Hono<{ Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: AnalyticsEngineDataset }; }>({ getPath: req => { @@ -80,7 +80,9 @@ const customLogger = (message: string, ...rest: string[]) => { app.use('*', logger(customLogger)); app.use('*', async (c, next) => { - console.log(`Hello from ⛅ ${c.req.raw.cf?.colo || 'UNK'}`); + if (c.req.raw.cf) { + console.log(`Hello from ⛅ ${c.req.raw.cf.colo || 'UNK'}`); + } console.log('userAgent', c.req.header('user-agent')); await next(); }); @@ -115,4 +117,4 @@ export default { }); } } -}; +}; \ No newline at end of file diff --git a/test/index.test.ts b/test/index.test.ts index 45185e4..7daa0fc 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -1,4 +1,4 @@ -import { cacheWrapper } from '../src/worker'; +import { app } from '../src/worker'; const botHeaders = { 'User-Agent': 'Discordbot/2.0' }; const humanHeaders = { @@ -29,13 +29,13 @@ if (!globalThis.performance.now) { } test('Home page redirect', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://fxtwitter.com', { method: 'GET', headers: botHeaders }) ); - const resultHuman = await cacheWrapper( + const resultHuman = await app.fetch( new Request('https://fxtwitter.com', { method: 'GET', headers: humanHeaders @@ -48,7 +48,7 @@ test('Home page redirect', async () => { }); test('Tweet redirect human', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: humanHeaders @@ -59,7 +59,7 @@ test('Tweet redirect human', async () => { }); test('Tweet redirect human custom base redirect', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: { @@ -73,7 +73,7 @@ test('Tweet redirect human custom base redirect', async () => { }); test('Twitter moment redirect', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request( 'https://fxtwitter.com/i/events/1572638642127966214?t=0UK7Ny-Jnsp-dUGzlb-M8w&s=35', { @@ -87,7 +87,7 @@ test('Twitter moment redirect', async () => { }); test('Tweet response robot', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: botHeaders @@ -97,7 +97,7 @@ test('Tweet response robot', async () => { }); test('API fetch basic Tweet', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://api.fxtwitter.com/status/20', { method: 'GET', headers: botHeaders @@ -132,7 +132,7 @@ test('API fetch basic Tweet', async () => { }); // test('API fetch video Tweet', async () => { -// const result = await cacheWrapper( +// const result = await app.fetch( // new Request('https://api.fxtwitter.com/X/status/854416760933556224', { // method: 'GET', // headers: botHeaders @@ -177,7 +177,7 @@ test('API fetch basic Tweet', async () => { // }); // test('API fetch multi-photo Tweet', async () => { -// const result = await cacheWrapper( +// const result = await app.fetch( // new Request('https://api.fxtwitter.com/Twitter/status/1445094085593866246', { // method: 'GET', // headers: botHeaders @@ -224,7 +224,7 @@ test('API fetch basic Tweet', async () => { // }); // test('API fetch poll Tweet', async () => { -// const result = await cacheWrapper( +// const result = await app.fetch( // new Request('https://api.fxtwitter.com/status/1055475950543167488', { // method: 'GET', // headers: botHeaders @@ -273,7 +273,7 @@ test('API fetch basic Tweet', async () => { // }); test('API fetch user', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://api.fxtwitter.com/x', { method: 'GET', headers: botHeaders @@ -303,7 +303,7 @@ test('API fetch user', async () => { }); test('API fetch user that does not exist', async () => { - const result = await cacheWrapper( + const result = await app.fetch( new Request('https://api.fxtwitter.com/usesaahah123', { method: 'GET', headers: botHeaders From 44ff4765cae44cf664a29e6038c9ec032db462f7 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 03:31:15 -0500 Subject: [PATCH 13/30] use nullish for this --- src/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.ts b/src/worker.ts index c0455ab..de6eddc 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -81,7 +81,7 @@ app.use('*', logger(customLogger)); app.use('*', async (c, next) => { if (c.req.raw.cf) { - console.log(`Hello from ⛅ ${c.req.raw.cf.colo || 'UNK'}`); + console.log(`Hello from ⛅ ${c.req.raw.cf.colo ?? 'UNK'}`); } console.log('userAgent', c.req.header('user-agent')); await next(); From feeaba5a64313e47a6b7aa93dc060f08d3d48e45 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 04:04:21 -0500 Subject: [PATCH 14/30] Move index.test.ts to worker.test.ts --- test/{index.test.ts => worker.test.ts} | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) rename test/{index.test.ts => worker.test.ts} (96%) diff --git a/test/index.test.ts b/test/worker.test.ts similarity index 96% rename from test/index.test.ts rename to test/worker.test.ts index 7daa0fc..89d0016 100644 --- a/test/index.test.ts +++ b/test/worker.test.ts @@ -29,13 +29,13 @@ if (!globalThis.performance.now) { } test('Home page redirect', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://fxtwitter.com', { method: 'GET', headers: botHeaders }) ); - const resultHuman = await app.fetch( + const resultHuman = await app.request( new Request('https://fxtwitter.com', { method: 'GET', headers: humanHeaders @@ -48,7 +48,7 @@ test('Home page redirect', async () => { }); test('Tweet redirect human', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: humanHeaders @@ -59,7 +59,7 @@ test('Tweet redirect human', async () => { }); test('Tweet redirect human custom base redirect', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: { @@ -73,7 +73,7 @@ test('Tweet redirect human custom base redirect', async () => { }); test('Twitter moment redirect', async () => { - const result = await app.fetch( + const result = await app.request( new Request( 'https://fxtwitter.com/i/events/1572638642127966214?t=0UK7Ny-Jnsp-dUGzlb-M8w&s=35', { @@ -87,7 +87,7 @@ test('Twitter moment redirect', async () => { }); test('Tweet response robot', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: botHeaders @@ -97,7 +97,7 @@ test('Tweet response robot', async () => { }); test('API fetch basic Tweet', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://api.fxtwitter.com/status/20', { method: 'GET', headers: botHeaders @@ -132,7 +132,7 @@ test('API fetch basic Tweet', async () => { }); // test('API fetch video Tweet', async () => { -// const result = await app.fetch( +// const result = await app.request( // new Request('https://api.fxtwitter.com/X/status/854416760933556224', { // method: 'GET', // headers: botHeaders @@ -177,7 +177,7 @@ test('API fetch basic Tweet', async () => { // }); // test('API fetch multi-photo Tweet', async () => { -// const result = await app.fetch( +// const result = await app.request( // new Request('https://api.fxtwitter.com/Twitter/status/1445094085593866246', { // method: 'GET', // headers: botHeaders @@ -224,7 +224,7 @@ test('API fetch basic Tweet', async () => { // }); // test('API fetch poll Tweet', async () => { -// const result = await app.fetch( +// const result = await app.request( // new Request('https://api.fxtwitter.com/status/1055475950543167488', { // method: 'GET', // headers: botHeaders @@ -273,7 +273,7 @@ test('API fetch basic Tweet', async () => { // }); test('API fetch user', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://api.fxtwitter.com/x', { method: 'GET', headers: botHeaders @@ -303,7 +303,7 @@ test('API fetch user', async () => { }); test('API fetch user that does not exist', async () => { - const result = await app.fetch( + const result = await app.request( new Request('https://api.fxtwitter.com/usesaahah123', { method: 'GET', headers: botHeaders From 1b27ded260030e1d5d6810da8db35ff38c10c506 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 04:39:15 -0500 Subject: [PATCH 15/30] Try to disable this and see if it still works --- src/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.ts b/src/worker.ts index de6eddc..39d8546 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -51,7 +51,7 @@ app.use( allowedSearchParams: /(.*)/ }, - integrations: [new RewriteFrames({ root: '/' })], + // integrations: [new RewriteFrames({ root: '/' }) as any], release: RELEASE_NAME }) ); From b4b1a28bfceb462046aef88074207476eea04c4b Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 04:46:10 -0500 Subject: [PATCH 16/30] Only use sentry if SENTRY_DSN specified --- jestconfig.json | 3 +-- src/worker.ts | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/jestconfig.json b/jestconfig.json index 19c2be5..771d3c8 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -20,6 +20,5 @@ "TEST": true }, "testRegex": "/test/.*\\.test\\.ts$", - "collectCoverageFrom": ["src/**/*.{ts,js}"], - "useESM": true + "collectCoverageFrom": ["src/**/*.{ts,js}"] } diff --git a/src/worker.ts b/src/worker.ts index 39d8546..ba6eb8c 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -42,19 +42,21 @@ export const app = new Hono<{ } }); -app.use( - '*', - sentry({ - dsn: SENTRY_DSN, - requestDataOptions: { - allowedHeaders: /(.*)/, - allowedSearchParams: /(.*)/ - }, +if (SENTRY_DSN) { + app.use( + '*', + sentry({ + dsn: SENTRY_DSN, + requestDataOptions: { + allowedHeaders: /(.*)/, + allowedSearchParams: /(.*)/ + }, - // integrations: [new RewriteFrames({ root: '/' }) as any], - release: RELEASE_NAME - }) -); + // integrations: [new RewriteFrames({ root: '/' }) as any], + release: RELEASE_NAME + }) + ); +} app.use('*', async (c, next) => { /* Apply all headers from Constants.RESPONSE_HEADERS */ From b70ad420c3a4890dd8ceda907af3839e04cdbe87 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 04:49:59 -0500 Subject: [PATCH 17/30] add null SENTRY_DSN to tests --- jestconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jestconfig.json b/jestconfig.json index 771d3c8..7e51963 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -17,7 +17,7 @@ "REDIRECT_URL": "https://github.com/FixTweet/FixTweet", "EMBED_URL": "https://github.com/FixTweet/FixTweet", "RELEASE_NAME": "fixtweet-test", - "TEST": true + "SENTRY_DSN": null }, "testRegex": "/test/.*\\.test\\.ts$", "collectCoverageFrom": ["src/**/*.{ts,js}"] From d509965a3d8881f02da4d06ab193fb75f8181f21 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 04:52:36 -0500 Subject: [PATCH 18/30] Don't let onError fail if no sentry --- src/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker.ts b/src/worker.ts index ba6eb8c..1523dba 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -67,7 +67,7 @@ app.use('*', async (c, next) => { }); app.onError((err, c) => { - c.get('sentry').captureException(err); + c.get('sentry')?.captureException?.(err); console.error(err.stack); c.status(200); c.header('cache-control', noCache); From 91cadb2fcac1c65b8004f3bbdbfd2dcd0e7615a2 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 05:02:39 -0500 Subject: [PATCH 19/30] Multiple fixes for testing --- src/caches.ts | 12 +++++++++++- src/fetch.ts | 10 +++++----- src/helpers/translate.ts | 4 ++-- src/providers/twitter/conversation.ts | 6 +++--- src/types/env.d.ts | 7 +------ src/worker.ts | 3 ++- 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/caches.ts b/src/caches.ts index 48c3938..3714735 100644 --- a/src/caches.ts +++ b/src/caches.ts @@ -17,7 +17,17 @@ export const cacheMiddleware = (): MiddlewareHandler => async (c, next) => { console.log('cacheUrl', cacheUrl); - const cacheKey = new Request(cacheUrl.toString(), request); + let cacheKey: Request; + + try { + cacheKey = new Request(cacheUrl.toString(), request); + } catch(e) { + /* In Miniflare, you can't really create requests like this, so we ignore caching in the test environment */ + await next(); + return c.res.clone(); + } + + const cache = caches.default; switch (request.method) { diff --git a/src/fetch.ts b/src/fetch.ts index 760f3fb..5d719ed 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -24,7 +24,7 @@ export const twitterFetch = async ( url: string, useElongator = experimentCheck( Experiment.ELONGATOR_BY_DEFAULT, - typeof c.env.TwitterProxy !== 'undefined' + typeof c.env?.TwitterProxy !== 'undefined' ), validateFunction: (response: unknown) => boolean, elongatorRequired = false @@ -141,10 +141,10 @@ export const twitterFetch = async ( let apiRequest; try { - if (useElongator && typeof c.env.TwitterProxy !== 'undefined') { + if (useElongator && typeof c.env?.TwitterProxy !== 'undefined') { console.log('Fetching using elongator'); const performanceStart = performance.now(); - apiRequest = await c.env.TwitterProxy.fetch(url, { + apiRequest = await c.env?.TwitterProxy.fetch(url, { method: 'GET', headers: headers }); @@ -189,7 +189,7 @@ export const twitterFetch = async ( if ( !wasElongatorDisabled && !useElongator && - typeof c.env.TwitterProxy !== 'undefined' && + typeof c.env?.TwitterProxy !== 'undefined' && (response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason === 'NsfwLoggedOut' ) { @@ -251,7 +251,7 @@ export const fetchUser = async ( c: Context, useElongator = experimentCheck( Experiment.ELONGATOR_PROFILE_API, - typeof c.env.TwitterProxy !== 'undefined' + typeof c.env?.TwitterProxy !== 'undefined' ) ): Promise => { return (await twitterFetch( diff --git a/src/helpers/translate.ts b/src/helpers/translate.ts index fcdc22d..e1368e0 100644 --- a/src/helpers/translate.ts +++ b/src/helpers/translate.ts @@ -46,14 +46,14 @@ export const translateTweet = async ( headers['x-twitter-client-language'] = language; /* As of August 2023, you can no longer fetch translations with guest token */ - if (typeof c.env.TwitterProxy === 'undefined') { + if (typeof c.env?.TwitterProxy === 'undefined') { return null; } try { const url = `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`; console.log(url, headers); - translationApiResponse = await c.env.TwitterProxy.fetch(url, { + translationApiResponse = await c.env?.TwitterProxy.fetch(url, { method: 'GET', headers: headers }); diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 5ee022f..da6bf9b 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -8,7 +8,7 @@ import { Context } from 'hono'; export const fetchTweetDetail = async ( c: Context, status: string, - useElongator = typeof c.env.TwitterProxy !== 'undefined', + useElongator = typeof c.env?.TwitterProxy !== 'undefined', cursor: string | null = null ): Promise => { return (await twitterFetch( @@ -93,7 +93,7 @@ export const fetchByRestId = async ( c: Context, useElongator = experimentCheck( Experiment.ELONGATOR_BY_DEFAULT, - typeof c.env.TwitterProxy !== 'undefined' + typeof c.env?.TwitterProxy !== 'undefined' ) ): Promise => { return (await twitterFetch( @@ -287,7 +287,7 @@ export const constructTwitterThread = async ( Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why. I'll figure out why eventually, but for now just don't use TweetDetail for this. */ if ( - typeof c.env.TwitterProxy !== 'undefined' && + typeof c.env?.TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread) ) { diff --git a/src/types/env.d.ts b/src/types/env.d.ts index c73b538..1e4051b 100644 --- a/src/types/env.d.ts +++ b/src/types/env.d.ts @@ -12,9 +12,4 @@ declare const MOSAIC_DOMAIN_LIST: string; declare const API_HOST_LIST: string; declare const SENTRY_DSN: string; -declare const RELEASE_NAME: string; - -declare const TEST: boolean | undefined; - -declare const TwitterProxy: Fetcher; -declare const AnalyticsEngine: AnalyticsEngineDataset; +declare const RELEASE_NAME: string; \ No newline at end of file diff --git a/src/worker.ts b/src/worker.ts index 1523dba..f679080 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -52,7 +52,8 @@ if (SENTRY_DSN) { allowedSearchParams: /(.*)/ }, - // integrations: [new RewriteFrames({ root: '/' }) as any], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + integrations: [new RewriteFrames({ root: '/' }) as any], release: RELEASE_NAME }) ); From e9528a7e00a6e3c48ba33e51d20ed69a1374c990 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 05:27:01 -0500 Subject: [PATCH 20/30] Maybe fix profile requests not working --- src/realms/twitter/router.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts index 748659e..8f340aa 100644 --- a/src/realms/twitter/router.ts +++ b/src/realms/twitter/router.ts @@ -28,6 +28,7 @@ export const getBaseRedirectUrl = (c: Context) => { /* Workaround for some dumb maybe-build time issue where statusRequest isn't ready or something because none of these trigger*/ const tweetRequest = async (c: Context) => await statusRequest(c); +const _profileRequest = async (c: Context) => await profileRequest(c); twitter.get('/:prefix?/:handle?/:endpoint{status(es)?}/:id/:language?', tweetRequest); twitter.get(':handle?/:endpoint{status(es)?}/:id/:language?', tweetRequest); @@ -49,6 +50,6 @@ twitter.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT)); twitter.get('/i/events/:id', genericTwitterRedirect); twitter.get('/hashtag/:hashtag', genericTwitterRedirect); -twitter.get('/:handle', profileRequest); +twitter.get('/:handle', _profileRequest); twitter.all('*', async c => c.redirect(Constants.REDIRECT_URL, 302)); From 3d536aa5f17110465cd581e916f9cb4f78449a98 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 05:33:04 -0500 Subject: [PATCH 21/30] Found the human redirect issue --- src/realms/twitter/routes/status.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts index 625cc8d..4e77279 100644 --- a/src/realms/twitter/routes/status.ts +++ b/src/realms/twitter/routes/status.ts @@ -141,8 +141,7 @@ export const statusRequest = async (c: Context) => { /* A human has clicked a fxtwitter.com/:screen_name/status/:id link! Obviously we just need to redirect to the Tweet directly.*/ console.log('Matched human UA', userAgent); - - c.redirect(`${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, 302); - return c.text(''); + + return c.redirect(`${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, 302); } }; From c937f8543ba12c76f90fbedd32f7242258747cea Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 05:36:13 -0500 Subject: [PATCH 22/30] Remove undocumented .json endpoint --- src/realms/twitter/routes/status.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts index 4e77279..0e7b827 100644 --- a/src/realms/twitter/routes/status.ts +++ b/src/realms/twitter/routes/status.ts @@ -80,12 +80,7 @@ export const statusRequest = async (c: Context) => { const baseUrl = getBaseRedirectUrl(c); - /* Check if request is to api.fxtwitter.com, or the tweet is appended with .json - Note that unlike TwitFix, FixTweet will never generate embeds for .json, and - in fact we only support .json because it's what people using TwitFix API would - be used to. */ if ( - url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null || Constants.API_HOST_LIST.includes(url.hostname) ) { console.log('JSON API request'); From 3b14d1e63f161ffd8b689e3354751ae93bd59493 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 15:48:59 -0500 Subject: [PATCH 23/30] Fix routing and api robots.txt --- src/fetch.ts | 8 -------- src/realms/api/router.ts | 4 ++-- src/realms/twitter/router.ts | 8 ++++---- src/realms/twitter/routes/status.ts | 3 ++- src/strings.ts | 4 +++- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index 5d719ed..15840f7 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -6,19 +6,12 @@ import { generateUserAgent } from './helpers/useragent'; const API_ATTEMPTS = 3; let wasElongatorDisabled = false; -/* TODO: Figure out why TS globals were broken when not forcing globalThis */ -declare const globalThis: { - fetchCompletedTime: number; -}; - const generateSnowflake = () => { const epoch = 1288834974657n; /* Twitter snowflake epoch */ const timestamp = BigInt(Date.now()) - epoch; return String((timestamp << 22n) | BigInt(Math.floor(Math.random() * 696969))); }; -globalThis.fetchCompletedTime = 0; - export const twitterFetch = async ( c: Context, url: string, @@ -184,7 +177,6 @@ export const twitterFetch = async ( continue; } - globalThis.fetchCompletedTime = performance.now(); if ( !wasElongatorDisabled && diff --git a/src/realms/api/router.ts b/src/realms/api/router.ts index f692ac8..a4fcc2c 100644 --- a/src/realms/api/router.ts +++ b/src/realms/api/router.ts @@ -9,8 +9,8 @@ export const api = new Hono(); /* Current v1 API endpoints. Currently, these still go through the Twitter embed requests. API v2+ won't do this. */ api.get('/status/:id/:language?', statusRequest); api.get('/:handle/status/:id/:language?', statusRequest); +api.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT_API)); api.get('/:handle', profileRequest); -api.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT)); -api.all('*', async c => c.redirect(Constants.API_DOCS_URL, 302)); +api.all('*', async c => c.redirect(Constants.API_DOCS_URL, 302)); \ No newline at end of file diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts index 8f340aa..7976119 100644 --- a/src/realms/twitter/router.ts +++ b/src/realms/twitter/router.ts @@ -30,14 +30,14 @@ export const getBaseRedirectUrl = (c: Context) => { const tweetRequest = async (c: Context) => await statusRequest(c); const _profileRequest = async (c: Context) => await profileRequest(c); -twitter.get('/:prefix?/:handle?/:endpoint{status(es)?}/:id/:language?', tweetRequest); -twitter.get(':handle?/:endpoint{status(es)?}/:id/:language?', tweetRequest); +twitter.get('/:handle{[0-9a-zA-Z_]+}/:endpoint{status(es)?}/:id{[0-9]+}/:language{[a-z]+}?', tweetRequest); +twitter.get('/:prefix{(dir|dl)}/:handle{[0-9a-zA-Z_]+}/status/:id{[0-9]+}/:language{[a-z]+}?', tweetRequest); twitter.get( - '/:prefix?/:handle/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + '/:handle{[0-9a-zA-Z_]+}/status/:id{[0-9]+}/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language{[a-z]+}?', tweetRequest ); twitter.get( - '/:handle/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?', + '/:prefix{(dir|dl)}/:handle{[0-9a-zA-Z_]+}/status/:id{[0-9]+}/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language{[a-z]+}?', tweetRequest ); diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts index 0e7b827..4b4e354 100644 --- a/src/realms/twitter/routes/status.ts +++ b/src/realms/twitter/routes/status.ts @@ -6,7 +6,8 @@ import { Strings } from '../../../strings'; /* Handler for status (Tweet) request */ export const statusRequest = async (c: Context) => { - const { handle, id, mediaNumber, language, prefix } = c.req.param(); + const { prefix, handle, id, mediaNumber, language } = c.req.param(); + console.log('req', JSON.stringify(c.req)) const url = new URL(c.req.url); const flags: InputFlags = {}; diff --git a/src/strings.ts b/src/strings.ts index 98c3eaa..a90b9d4 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -228,5 +228,7 @@ Disallow: /owoembed/ Allow: /watch?v=dQw4w9WgXcQ # 0100011101101111011011110110010000100000011000100110111101110100`, - X_DOMAIN_NOTICE: 'FixTweet - 🆕 x.com link? Try fixupx.com' + ROBOTS_TXT_API: `# Crawlers should not crawl API endpoints +User-agent: * +Disallow: /` }; From 1caa109c2f9822d2d7f22f5efd62ba9a155df33a Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 15:56:57 -0500 Subject: [PATCH 24/30] Try bumping node to 18 --- .github/workflows/build.yml | 2 +- .github/workflows/eslint.yml | 2 +- .github/workflows/tests.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e18e245..e29fb7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '18' cache: 'npm' cache-dependency-path: package-lock.json - run: npm install diff --git a/.github/workflows/eslint.yml b/.github/workflows/eslint.yml index 958418c..d847e9f 100644 --- a/.github/workflows/eslint.yml +++ b/.github/workflows/eslint.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 18 cache: npm cache-dependency-path: package-lock.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b7e3a7f..0d7ec15 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,8 +13,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '18' cache: 'npm' cache-dependency-path: package-lock.json - run: npm install - - run: npm run build && npm test + - run: npm test From e2489c23722ca08697c8c9b2fea5459adacd5954 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 15:58:04 -0500 Subject: [PATCH 25/30] Yeah no we still need build to test on gh actions --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d7ec15..974e5e6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,4 +17,4 @@ jobs: cache: 'npm' cache-dependency-path: package-lock.json - run: npm install - - run: npm test + - run: npm run build && npm test From ed14bac44c3807b93cda8a3c8643ee9dce510d29 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 16:13:08 -0500 Subject: [PATCH 26/30] Just checking to be sure this does not impact test env --- jestconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jestconfig.json b/jestconfig.json index 7e51963..447aa81 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -20,5 +20,6 @@ "SENTRY_DSN": null }, "testRegex": "/test/.*\\.test\\.ts$", - "collectCoverageFrom": ["src/**/*.{ts,js}"] + "collectCoverageFrom": ["src/**/*.{ts,js}"], + "useESM": true } From 1c8c71b0cd16a12ca8033389a1bd6811b201b149 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 16:14:28 -0500 Subject: [PATCH 27/30] Revert "Just checking to be sure this does not impact test env" This reverts commit ed14bac44c3807b93cda8a3c8643ee9dce510d29. --- jestconfig.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jestconfig.json b/jestconfig.json index 447aa81..7e51963 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -20,6 +20,5 @@ "SENTRY_DSN": null }, "testRegex": "/test/.*\\.test\\.ts$", - "collectCoverageFrom": ["src/**/*.{ts,js}"], - "useESM": true + "collectCoverageFrom": ["src/**/*.{ts,js}"] } From 7dac72df79abc65eee65f636be3e3053c20396be Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 16:22:49 -0500 Subject: [PATCH 28/30] Add crash safety around executionCtx --- src/caches.ts | 1 + src/fetch.ts | 32 ++++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/caches.ts b/src/caches.ts index 3714735..89c7a50 100644 --- a/src/caches.ts +++ b/src/caches.ts @@ -56,6 +56,7 @@ export const cacheMiddleware = (): MiddlewareHandler => async (c, next) => { Use waitUntil so you can return the response without blocking on writing to cache */ try { + c.executionCtx && c.executionCtx.waitUntil(cache.put(cacheKey, response.clone())); } catch (error) { console.error((error as Error).stack); diff --git a/src/fetch.ts b/src/fetch.ts index 15840f7..0f3e4d0 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -163,11 +163,15 @@ export const twitterFetch = async ( console.log('Tweet was not found'); return {}; } - !useElongator && - c.executionCtx && - c.executionCtx.waitUntil( - cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) - ); + try{ + !useElongator && + c.executionCtx && + c.executionCtx.waitUntil( + cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) + ); + } catch (error) { + console.error((error as Error).stack); + } if (useElongator) { console.log('Elongator request failed, trying again without it'); wasElongatorDisabled = true; @@ -195,10 +199,14 @@ export const twitterFetch = async ( /* Running out of requests within our rate limit, let's purge the cache */ if (!useElongator && remainingRateLimit < 10) { console.log(`Purging token on this edge due to low rate limit remaining`); - c.executionCtx && - c.executionCtx.waitUntil( - cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) - ); + try { + c.executionCtx && + c.executionCtx.waitUntil( + cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) + ); + } catch (error) { + console.error((error as Error).stack); + } } if (!validateFunction(response)) { @@ -224,7 +232,11 @@ export const twitterFetch = async ( } }); console.log('Caching guest token'); - c.executionCtx.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); + try { + c.executionCtx.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); + } catch (error) { + console.error((error as Error).stack); + } } // @ts-expect-error - We'll pin the guest token to whatever response we have From ebb5e5e60d16dde68e23adeed044369ba08dae7c Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 16:24:47 -0500 Subject: [PATCH 29/30] Fix said crash safety --- src/fetch.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index 0f3e4d0..4056aab 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -223,20 +223,20 @@ export const twitterFetch = async ( newTokenGenerated = true; continue; } - /* If we've generated a new token, we'll cache it */ - if (c.executionCtx && newTokenGenerated && activate) { - const cachingResponse = new Response(await activate.clone().text(), { - headers: { - ...tokenHeaders, - 'cache-control': `max-age=${Constants.GUEST_TOKEN_MAX_AGE}` - } - }); - console.log('Caching guest token'); - try { + try { + /* If we've generated a new token, we'll cache it */ + if (c.executionCtx && newTokenGenerated && activate) { + const cachingResponse = new Response(await activate.clone().text(), { + headers: { + ...tokenHeaders, + 'cache-control': `max-age=${Constants.GUEST_TOKEN_MAX_AGE}` + } + }); + console.log('Caching guest token'); c.executionCtx.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); - } catch (error) { - console.error((error as Error).stack); } + } catch (error) { + console.error((error as Error).stack); } // @ts-expect-error - We'll pin the guest token to whatever response we have From c75e5191fad1e0ded289a31ecf7ce086f5db171b Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Fri, 10 Nov 2023 16:30:05 -0500 Subject: [PATCH 30/30] Try bumping to node 20 --- .github/workflows/build.yml | 2 +- .github/workflows/eslint.yml | 2 +- .github/workflows/tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e29fb7d..9fc87a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'npm' cache-dependency-path: package-lock.json - run: npm install diff --git a/.github/workflows/eslint.yml b/.github/workflows/eslint.yml index d847e9f..e6ec16f 100644 --- a/.github/workflows/eslint.yml +++ b/.github/workflows/eslint.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 18 + node-version: 20 cache: npm cache-dependency-path: package-lock.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 974e5e6..082a578 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' cache: 'npm' cache-dependency-path: package-lock.json - run: npm install