diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e18e245..9fc87a2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '20' cache: 'npm' cache-dependency-path: package-lock.json - run: npm install diff --git a/.github/workflows/eslint.yml b/.github/workflows/eslint.yml index 958418c..e6ec16f 100644 --- a/.github/workflows/eslint.yml +++ b/.github/workflows/eslint.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: 16 + node-version: 20 cache: npm cache-dependency-path: package-lock.json diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b7e3a7f..082a578 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '20' cache: 'npm' cache-dependency-path: package-lock.json - run: npm install diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 82f22d9..57ede78 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -60,8 +60,9 @@ await esbuild.build({ entryPoints: ['src/worker.ts'], sourcemap: 'external', outdir: 'dist', - minify: true, + minify: false, bundle: true, + format: 'esm', plugins: [ sentryEsbuildPlugin({ org: process.env.SENTRY_ORG, diff --git a/jestconfig.json b/jestconfig.json index 19c2be5..7e51963 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -17,9 +17,8 @@ "REDIRECT_URL": "https://github.com/FixTweet/FixTweet", "EMBED_URL": "https://github.com/FixTweet/FixTweet", "RELEASE_NAME": "fixtweet-test", - "TEST": true + "SENTRY_DSN": null }, "testRegex": "/test/.*\\.test\\.ts$", - "collectCoverageFrom": ["src/**/*.{ts,js}"], - "useESM": true + "collectCoverageFrom": ["src/**/*.{ts,js}"] } diff --git a/package-lock.json b/package-lock.json index 2697a09..6b6e546 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,8 +9,8 @@ "version": "1.0.0", "license": "MIT", "dependencies": { - "itty-router": "^4.0.23", - "toucan-js": "^3.3.1" + "@hono/sentry": "^1.0.0", + "hono": "^3.9.2" }, "devDependencies": { "@cloudflare/workers-types": "^4.20231025.0", @@ -1240,6 +1240,17 @@ "node": ">=14" } }, + "node_modules/@hono/sentry": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@hono/sentry/-/sentry-1.0.0.tgz", + "integrity": "sha512-GbPxgpGuasM2zRCSaA77MPWu4KDcuk/EMf7JJykjCvnOTbjmtr7FovNxsvg7xlXCIjZDgLmqBaoJMi3AxbeIAA==", + "dependencies": { + "toucan-js": "^3.2.2" + }, + "peerDependencies": { + "hono": "3.*" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.13", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz", @@ -4255,6 +4266,14 @@ "node": ">= 0.4" } }, + "node_modules/hono": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/hono/-/hono-3.9.2.tgz", + "integrity": "sha512-180NOiMadqU3lGmN6ajPDZvZPWus3a9mtVaAUR9uG0SImngBwRLA8vbnV0oUfUAgFT4nX55sGV9dVA06OuikHA==", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -4540,11 +4559,6 @@ "node": ">=8" } }, - "node_modules/itty-router": { - "version": "4.0.23", - "resolved": "https://registry.npmjs.org/itty-router/-/itty-router-4.0.23.tgz", - "integrity": "sha512-tP1NI8PVK43vWlBnIPqj47ni5FDSczFviA4wgBznscndo8lEvBA+pO3DD1rNbIQPcZhprr775iUTunyGvQMcBw==" - }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", diff --git a/package.json b/package.json index 8560606..8e96b9b 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "wrangler": "^3.15.0" }, "dependencies": { - "itty-router": "^4.0.23", - "toucan-js": "^3.3.1" + "@hono/sentry": "^1.0.0", + "hono": "^3.9.2" } } diff --git a/src/caches.ts b/src/caches.ts new file mode 100644 index 0000000..89c7a50 --- /dev/null +++ b/src/caches.ts @@ -0,0 +1,90 @@ +import { MiddlewareHandler } from 'hono'; +import { Constants } from './constants'; +import {} from 'hono'; + +/* Wrapper to handle caching, and misc things like catching robots.txt */ +export const cacheMiddleware = (): MiddlewareHandler => async (c, next) => { + const request = c.req; + const userAgent = request.header('User-Agent') ?? ''; + // https://developers.cloudflare.com/workers/examples/cache-api/ + const cacheUrl = new URL( + userAgent.includes('Telegram') + ? `${request.url}&telegram` + : userAgent.includes('Discord') + ? `${request.url}&discord` + : request.url + ); + + console.log('cacheUrl', cacheUrl); + + let cacheKey: Request; + + try { + cacheKey = new Request(cacheUrl.toString(), request); + } catch(e) { + /* In Miniflare, you can't really create requests like this, so we ignore caching in the test environment */ + await next(); + return c.res.clone(); + } + + + const cache = caches.default; + + switch (request.method) { + case 'GET': + if ( + !Constants.API_HOST_LIST.includes(cacheUrl.hostname) && + !request.header('Cookie')?.includes('base_redirect') + ) { + /* cache may be undefined in tests */ + const cachedResponse = await cache.match(cacheKey); + + if (cachedResponse) { + console.log('Cache hit'); + return new Response(cachedResponse.body, cachedResponse); + } + + console.log('Cache miss'); + } + + await next(); + + // eslint-disable-next-line no-case-declarations + const response = c.res.clone(); + + /* Store the fetched response as cacheKey + Use waitUntil so you can return the response without blocking on + writing to cache */ + try { + c.executionCtx && + c.executionCtx.waitUntil(cache.put(cacheKey, response.clone())); + } catch (error) { + console.error((error as Error).stack); + } + + return response; + /* Telegram sends this from Webpage Bot, and Cloudflare sends it if we purge cache, and we respect it. + PURGE is not defined in an RFC, but other servers like Nginx apparently use it. + + Update 2023-11-09: + + For some reason, even before migrating to Hono, this returns 403 Forbidden now when PURGEd. + I'm not sure why, as this is clearly not what we are doing. Is Cloudflare doing this? Is something else wrong? We'll also accept DELETE to do the same I guess. */ + case 'PURGE': + case 'DELETE': + console.log('Purging cache as requested'); + await cache.delete(cacheKey); + return c.text(''); + /* yes, we do give HEAD */ + case 'HEAD': + return c.text(''); + /* We properly state our OPTIONS when asked */ + case 'OPTIONS': + c.header('allow', Constants.RESPONSE_HEADERS.allow); + c.status(204); + return c.text(''); + default: + c.status(405); + return c.text(''); + } +}; diff --git a/src/constants.ts b/src/constants.ts index 0b0bfa5..0c2c911 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -59,8 +59,9 @@ export const Constants = { RESPONSE_HEADERS: { 'allow': 'OPTIONS, GET, PURGE, HEAD', 'content-type': 'text/html;charset=UTF-8', - 'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`, - 'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets + 'x-powered-by': `${RELEASE_NAME}`, + 'x-trans-rights': 'true', + 'Vary': 'Accept-Encoding, User-Agent' }, API_RESPONSE_HEADERS: { 'access-control-allow-origin': '*', diff --git a/src/embed/status.ts b/src/embed/status.ts index f968bae..e4f39d3 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -7,45 +7,43 @@ import { renderPhoto } from '../render/photo'; import { renderVideo } from '../render/video'; import { renderInstantView } from '../render/instantview'; import { constructTwitterThread } from '../providers/twitter/conversation'; -import { IRequest } from 'itty-router'; +import { Context } from 'hono'; -export const returnError = (error: string): StatusResponse => { - return { - text: Strings.BASE_HTML.format({ +export const returnError = (c: Context, error: string): Response => { + return c.text( + Strings.BASE_HTML.format({ lang: '', headers: [ ``, `` ].join('') }) - }; + ); }; - /* Handler for Twitter statuses (Tweets). Like Twitter, we use the terminologies interchangably. */ export const handleStatus = async ( + c: Context, status: string, mediaNumber: number | undefined, userAgent: string, flags: InputFlags, - language: string, - event: FetchEvent, - request: IRequest + language: string // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { console.log('Direct?', flags?.direct); let fetchWithThreads = false; /* TODO: Enable actually pulling threads once we can actually do something with them */ - if (request?.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) { + if (c.req.header('user-agent')?.includes('Telegram') && !flags?.direct) { fetchWithThreads = false; } const thread = await constructTwitterThread( status, fetchWithThreads, - request, + c, language, flags?.api ?? false ); @@ -76,27 +74,27 @@ export const handleStatus = async ( /* Catch this request if it's an API response */ if (flags?.api) { - return { - response: new Response(JSON.stringify(api), { - headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }, - status: api.code - }) - }; + c.status(api.code); + // Add every header from Constants.API_RESPONSE_HEADERS + for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { + c.header(header, value); + } + return c.json(api); } if (tweet === null) { - return returnError(Strings.ERROR_TWEET_NOT_FOUND); + return returnError(c, Strings.ERROR_TWEET_NOT_FOUND); } /* If there was any errors fetching the Tweet, we'll return it */ switch (api.code) { case 401: - return returnError(Strings.ERROR_PRIVATE); + return returnError(c, Strings.ERROR_PRIVATE); case 404: - return returnError(Strings.ERROR_TWEET_NOT_FOUND); + return returnError(c, Strings.ERROR_TWEET_NOT_FOUND); case 500: console.log(api); - return returnError(Strings.ERROR_API_FAIL); + return returnError(c, Strings.ERROR_API_FAIL); } const isTelegram = (userAgent || '').indexOf('Telegram') > -1; @@ -146,7 +144,7 @@ export const handleStatus = async ( } if (redirectUrl) { - return { response: Response.redirect(redirectUrl, 302) }; + return c.redirect(redirectUrl, 302); } } @@ -157,7 +155,6 @@ export const handleStatus = async ( const engagementText = authorText.replace(/ {4}/g, ' '); let siteName = Constants.BRANDING_NAME; let newText = tweet.text; - let cacheControl: string | null = null; /* Base headers included in all responses */ const headers = [ @@ -217,7 +214,8 @@ export const handleStatus = async ( console.log('overrideMedia', JSON.stringify(overrideMedia)); if (!flags?.textOnly) { - const media = tweet.media?.all && tweet.media?.all.length > 0 ? tweet.media : tweet.quote?.media || {} + const media = + tweet.media?.all && tweet.media?.all.length > 0 ? tweet.media : tweet.quote?.media || {}; if (overrideMedia) { let instructions: ResponseInstructions; @@ -340,7 +338,7 @@ export const handleStatus = async ( Yes, checking if this is a string is a hacky way to do this, but it can do it in way less code than actually comparing dates */ if (poll.time_left_en !== 'Final results') { - cacheControl = Constants.POLL_TWEET_CACHE; + c.header('cache-control', Constants.POLL_TWEET_CACHE); } /* And now we'll put the poll right after the Tweet text! */ @@ -427,12 +425,11 @@ export const handleStatus = async ( const lang = tweet.lang === null ? 'en' : tweet.lang || 'en'; /* Finally, after all that work we return the response HTML! */ - return { - text: Strings.BASE_HTML.format({ + return c.html( + Strings.BASE_HTML.format({ lang: `lang="${lang}"`, headers: headers.join(''), body: ivbody - }).replace(/>(\s+)<'), // Remove whitespace between tags - cacheControl: cacheControl - }; + }).replace(/>(\s+)<') + ); }; diff --git a/src/fetch.ts b/src/fetch.ts index 5973279..4056aab 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,3 +1,4 @@ +import { Context } from 'hono'; import { Constants } from './constants'; import { Experiment, experimentCheck } from './experiments'; import { generateUserAgent } from './helpers/useragent'; @@ -5,25 +6,18 @@ import { generateUserAgent } from './helpers/useragent'; const API_ATTEMPTS = 3; let wasElongatorDisabled = false; -/* TODO: Figure out why TS globals were broken when not forcing globalThis */ -declare const globalThis: { - fetchCompletedTime: number; -}; - const generateSnowflake = () => { const epoch = 1288834974657n; /* Twitter snowflake epoch */ const timestamp = BigInt(Date.now()) - epoch; return String((timestamp << 22n) | BigInt(Math.floor(Math.random() * 696969))); }; -globalThis.fetchCompletedTime = 0; - export const twitterFetch = async ( + c: Context, url: string, - event: FetchEvent, useElongator = experimentCheck( Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' + typeof c.env?.TwitterProxy !== 'undefined' ), validateFunction: (response: unknown) => boolean, elongatorRequired = false @@ -140,10 +134,10 @@ export const twitterFetch = async ( let apiRequest; try { - if (useElongator && typeof TwitterProxy !== 'undefined') { + if (useElongator && typeof c.env?.TwitterProxy !== 'undefined') { console.log('Fetching using elongator'); const performanceStart = performance.now(); - apiRequest = await TwitterProxy.fetch(url, { + apiRequest = await c.env?.TwitterProxy.fetch(url, { method: 'GET', headers: headers }); @@ -169,9 +163,15 @@ export const twitterFetch = async ( console.log('Tweet was not found'); return {}; } - !useElongator && - event && - event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); + try{ + !useElongator && + c.executionCtx && + c.executionCtx.waitUntil( + cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) + ); + } catch (error) { + console.error((error as Error).stack); + } if (useElongator) { console.log('Elongator request failed, trying again without it'); wasElongatorDisabled = true; @@ -181,12 +181,11 @@ export const twitterFetch = async ( continue; } - globalThis.fetchCompletedTime = performance.now(); if ( !wasElongatorDisabled && !useElongator && - typeof TwitterProxy !== 'undefined' && + typeof c.env?.TwitterProxy !== 'undefined' && (response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason === 'NsfwLoggedOut' ) { @@ -200,8 +199,14 @@ export const twitterFetch = async ( /* Running out of requests within our rate limit, let's purge the cache */ if (!useElongator && remainingRateLimit < 10) { console.log(`Purging token on this edge due to low rate limit remaining`); - event && - event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); + try { + c.executionCtx && + c.executionCtx.waitUntil( + cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }) + ); + } catch (error) { + console.error((error as Error).stack); + } } if (!validateFunction(response)) { @@ -218,16 +223,20 @@ export const twitterFetch = async ( newTokenGenerated = true; continue; } - /* If we've generated a new token, we'll cache it */ - if (event && newTokenGenerated && activate) { - const cachingResponse = new Response(await activate.clone().text(), { - headers: { - ...tokenHeaders, - 'cache-control': `max-age=${Constants.GUEST_TOKEN_MAX_AGE}` - } - }); - console.log('Caching guest token'); - event.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); + try { + /* If we've generated a new token, we'll cache it */ + if (c.executionCtx && newTokenGenerated && activate) { + const cachingResponse = new Response(await activate.clone().text(), { + headers: { + ...tokenHeaders, + 'cache-control': `max-age=${Constants.GUEST_TOKEN_MAX_AGE}` + } + }); + console.log('Caching guest token'); + c.executionCtx.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse)); + } + } catch (error) { + console.error((error as Error).stack); } // @ts-expect-error - We'll pin the guest token to whatever response we have @@ -243,13 +252,14 @@ export const twitterFetch = async ( export const fetchUser = async ( username: string, - event: FetchEvent, + c: Context, useElongator = experimentCheck( Experiment.ELONGATOR_PROFILE_API, - typeof TwitterProxy !== 'undefined' + typeof c.env?.TwitterProxy !== 'undefined' ) ): Promise => { return (await twitterFetch( + c, `${ Constants.TWITTER_ROOT }/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?variables=${encodeURIComponent( @@ -266,7 +276,6 @@ export const fetchUser = async ( verified_phone_label_enabled: true }) )}`, - event, useElongator, // Validator function (_res: unknown) => { diff --git a/src/helpers/graphql.ts b/src/helpers/graphql.ts index e11097e..0e10623 100644 --- a/src/helpers/graphql.ts +++ b/src/helpers/graphql.ts @@ -16,7 +16,9 @@ export const isGraphQLTweet = (response: unknown): response is GraphQLTweet => { return ( typeof response === 'object' && response !== null && - // @ts-expect-error it's 6 am please let me sleep - ('__typename' in response && (response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults') || ('legacy' in response && response.legacy?.full_text)) + (('__typename' in response && + (response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults')) || + // @ts-expect-error it's 6 am please let me sleep + ('legacy' in response && response.legacy?.full_text)) ); }; diff --git a/src/helpers/translate.ts b/src/helpers/translate.ts index a4f619e..e1368e0 100644 --- a/src/helpers/translate.ts +++ b/src/helpers/translate.ts @@ -1,10 +1,12 @@ +import { Context } from 'hono'; import { Constants } from '../constants'; /* Handles translating Tweets when asked! */ export const translateTweet = async ( tweet: GraphQLTweet, guestToken: string, - language: string + language: string, + c: Context ): Promise => { const csrfToken = crypto.randomUUID().replace(/-/g, ''); // Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match @@ -44,14 +46,14 @@ export const translateTweet = async ( headers['x-twitter-client-language'] = language; /* As of August 2023, you can no longer fetch translations with guest token */ - if (typeof TwitterProxy === 'undefined') { + if (typeof c.env?.TwitterProxy === 'undefined') { return null; } try { const url = `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`; console.log(url, headers); - translationApiResponse = await TwitterProxy.fetch(url, { + translationApiResponse = await c.env?.TwitterProxy.fetch(url, { method: 'GET', headers: headers }); diff --git a/src/helpers/useragent.ts b/src/helpers/useragent.ts index d09d536..42788f4 100644 --- a/src/helpers/useragent.ts +++ b/src/helpers/useragent.ts @@ -1,6 +1,6 @@ /* We keep this value up-to-date for making our requests to Twitter as indistinguishable from normal user traffic as possible. */ -const fakeChromeVersion = 118; +const fakeChromeVersion = 119; const platformWindows = 'Windows NT 10.0; Win64; x64'; const platformMac = 'Macintosh; Intel Mac OS X 10_15_7'; const platformLinux = 'X11; Linux x86_64'; diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index 6b86c01..fc04bfd 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -32,7 +32,6 @@ export const truncateWithEllipsis = (str: string, maxLength: number): string => return truncated.length < str.length ? truncated + '…' : truncated; }; - const numberFormat = new Intl.NumberFormat('en-US'); export const formatNumber = (num: number) => numberFormat.format(num); diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 59381ff..da6bf9b 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -1,17 +1,18 @@ -import { IRequest } from 'itty-router'; import { Constants } from '../../constants'; import { twitterFetch } from '../../fetch'; import { buildAPITweet } from './processor'; import { Experiment, experimentCheck } from '../../experiments'; import { isGraphQLTweet } from '../../helpers/graphql'; +import { Context } from 'hono'; export const fetchTweetDetail = async ( + c: Context, status: string, - event: FetchEvent, - useElongator = typeof TwitterProxy !== 'undefined', + useElongator = typeof c.env?.TwitterProxy !== 'undefined', cursor: string | null = null ): Promise => { return (await twitterFetch( + c, `${ Constants.TWITTER_ROOT }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( @@ -56,7 +57,6 @@ export const fetchTweetDetail = async ( withArticleRichContentState: true }) )}`, - event, useElongator, (_conversation: unknown) => { const conversation = _conversation as TweetDetailResult; @@ -90,13 +90,14 @@ export const fetchTweetDetail = async ( export const fetchByRestId = async ( status: string, - event: FetchEvent, + c: Context, useElongator = experimentCheck( Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' + typeof c.env?.TwitterProxy !== 'undefined' ) ): Promise => { return (await twitterFetch( + c, `${ Constants.TWITTER_ROOT }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( @@ -133,7 +134,6 @@ export const fetchByRestId = async ( withArticleRichContentState: true }) )}`, - event, useElongator, (_conversation: unknown) => { const conversation = _conversation as TweetResultsByRestIdResult; @@ -273,7 +273,7 @@ const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { export const constructTwitterThread = async ( id: string, processThread = false, - request: IRequest, + c: Context, language: string | undefined, legacyAPI = false ): Promise => { @@ -286,9 +286,13 @@ export const constructTwitterThread = async ( Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why. I'll figure out why eventually, but for now just don't use TweetDetail for this. */ - if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { + if ( + typeof c.env?.TwitterProxy !== 'undefined' && + !language && + (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread) + ) { console.log('Using TweetDetail for request...'); - response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult; + response = (await fetchTweetDetail(c, id)) as TweetDetailResult; console.log(response); @@ -312,7 +316,7 @@ export const constructTwitterThread = async ( /* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */ if (!response) { console.log('Using TweetResultsByRestId for request...'); - response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult; + response = (await fetchByRestId(id, c)) as TweetResultsByRestIdResult; const result = response?.data?.tweetResult?.result as GraphQLTweet; @@ -320,7 +324,7 @@ export const constructTwitterThread = async ( return { post: null, thread: null, author: null, code: 404 }; } - const buildPost = await buildAPITweet(result, language, false, legacyAPI); + const buildPost = await buildAPITweet(c, result, language, false, legacyAPI); if ((buildPost as FetchResults)?.status === 401) { return { post: null, thread: null, author: null, code: 401 }; @@ -343,7 +347,7 @@ export const constructTwitterThread = async ( return { post: null, thread: null, author: null, code: 404 }; } - post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet; + post = (await buildAPITweet(c, originalTweet, undefined, false, legacyAPI)) as APITweet; if (post === null) { return { post: null, thread: null, author: null, code: 404 }; @@ -400,7 +404,7 @@ export const constructTwitterThread = async ( let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); + loadCursor = await fetchTweetDetail(c, id, true, cursor.value); if ( typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === @@ -464,7 +468,7 @@ export const constructTwitterThread = async ( let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); + loadCursor = await fetchTweetDetail(c, id, true, cursor.value); if ( typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === @@ -501,18 +505,21 @@ export const constructTwitterThread = async ( }; threadTweets.forEach(async tweet => { - socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet); + socialThread.thread?.push((await buildAPITweet(c, tweet, undefined, true, false)) as APITweet); }); return socialThread; }; -export const threadAPIProvider = async (request: IRequest) => { - const { id } = request.params; +export const threadAPIProvider = async (c: Context) => { + const id = c.req.param('id') as string; - const processedResponse = await constructTwitterThread(id, true, request, undefined); + const processedResponse = await constructTwitterThread(id, true, c, undefined); - return new Response(JSON.stringify(processedResponse), { - headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } - }); + c.status(processedResponse.code); + // Add every header from Constants.API_RESPONSE_HEADERS + for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { + c.header(header, value); + } + return c.json(processedResponse); }; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index 62e7b76..58d140b 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -7,8 +7,10 @@ import { unescapeText } from '../../helpers/utils'; import { processMedia } from '../../helpers/media'; import { convertToApiUser } from './profile'; import { translateTweet } from '../../helpers/translate'; +import { Context } from 'hono'; export const buildAPITweet = async ( + c: Context, tweet: GraphQLTweet, language: string | undefined, threadPiece = false, @@ -149,19 +151,13 @@ export const buildAPITweet = async ( /* We found a quote tweet, let's process that too */ const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { - const buildQuote = (await buildAPITweet( - quoteTweet, - language, - threadPiece, - legacyAPI - )); + const buildQuote = await buildAPITweet(c, quoteTweet, language, threadPiece, legacyAPI); if ((buildQuote as FetchResults).status) { - apiTweet.quote = undefined + apiTweet.quote = undefined; } else { apiTweet.quote = buildQuote as APITweet; } - /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ if (apiTweet.embed_card === 'tweet' && typeof apiTweet.quote !== 'undefined') { apiTweet.embed_card = apiTweet.quote.embed_card; @@ -237,12 +233,12 @@ export const buildAPITweet = async ( apiTweet.embed_card = 'player'; } - console.log('language?', language) + console.log('language?', language); /* If a language is specified in API or by user, let's try translating it! */ if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { console.log(`Attempting to translate Tweet to ${language}...`); - const translateAPI = await translateTweet(tweet, '', language); + const translateAPI = await translateTweet(tweet, '', language, c); if (translateAPI !== null && translateAPI?.translation) { apiTweet.translation = { text: unescapeText( diff --git a/src/providers/twitter/profile.ts b/src/providers/twitter/profile.ts index fffb15f..0f4ad81 100644 --- a/src/providers/twitter/profile.ts +++ b/src/providers/twitter/profile.ts @@ -1,3 +1,4 @@ +import { Context } from 'hono'; import { Constants } from '../../constants'; import { fetchUser } from '../../fetch'; import { linkFixer } from '../../helpers/linkFixer'; @@ -79,10 +80,10 @@ const populateUserProperties = async ( available for free using api.fxtwitter.com. */ export const userAPI = async ( username: string, - event: FetchEvent + c: Context // flags?: InputFlags ): Promise => { - const userResponse = await fetchUser(username, event); + const userResponse = await fetchUser(username, c); if (!userResponse || !Object.keys(userResponse).length) { return { code: 404, diff --git a/src/realms/api/router.ts b/src/realms/api/router.ts new file mode 100644 index 0000000..a4fcc2c --- /dev/null +++ b/src/realms/api/router.ts @@ -0,0 +1,16 @@ +import { Hono } from 'hono'; +import { statusRequest } from '../twitter/routes/status'; +import { profileRequest } from '../twitter/routes/profile'; +import { Strings } from '../../strings'; +import { Constants } from '../../constants'; + +export const api = new Hono(); + +/* Current v1 API endpoints. Currently, these still go through the Twitter embed requests. API v2+ won't do this. */ +api.get('/status/:id/:language?', statusRequest); +api.get('/:handle/status/:id/:language?', statusRequest); +api.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT_API)); + +api.get('/:handle', profileRequest); + +api.all('*', async c => c.redirect(Constants.API_DOCS_URL, 302)); \ No newline at end of file diff --git a/src/realms/common/version.ts b/src/realms/common/version.ts new file mode 100644 index 0000000..da8e0c1 --- /dev/null +++ b/src/realms/common/version.ts @@ -0,0 +1,22 @@ +import { Context } from 'hono'; +import { sanitizeText } from '../../helpers/utils'; +import { Strings } from '../../strings'; + +export const versionRoute = async (c: Context) => { + c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate'); + const req = c.req; + return c.html( + Strings.VERSION_HTML.format({ + rtt: req.raw.cf?.clientTcpRtt ? `🏓 ${req.raw.cf.clientTcpRtt} ms RTT` : '', + colo: (req.raw.cf?.colo as string) ?? '??', + httpversion: (req.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', + tlsversion: (req.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version', + ip: req.header('x-real-ip') ?? req.header('cf-connecting-ip') ?? 'Unknown IP', + city: (req.raw.cf?.city as string) ?? 'Unknown City', + region: (req.raw.cf?.region as string) ?? req.raw.cf?.country ?? 'Unknown Region', + country: (req.raw.cf?.country as string) ?? 'Unknown Country', + asn: `AS${req.raw.cf?.asn ?? '??'} (${req.raw.cf?.asOrganization ?? 'Unknown ASN'})`, + ua: sanitizeText(req.header('user-agent') ?? 'Unknown User Agent') + }) + ); +}; diff --git a/src/realms/twitter/router.ts b/src/realms/twitter/router.ts new file mode 100644 index 0000000..7976119 --- /dev/null +++ b/src/realms/twitter/router.ts @@ -0,0 +1,55 @@ +import { Context, Hono } from 'hono'; +// import { cache } from "hono/cache"; +import { versionRoute } from '../common/version'; +import { Strings } from '../../strings'; +import { Constants } from '../../constants'; +import { genericTwitterRedirect, setRedirectRequest } from './routes/redirects'; +import { profileRequest } from './routes/profile'; +import { statusRequest } from './routes/status'; +import { oembed } from './routes/oembed'; + +export const twitter = new Hono(); + +export const getBaseRedirectUrl = (c: Context) => { + const baseRedirect = c.req.header('cookie')?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0]; + + if (baseRedirect) { + console.log('Found base redirect', baseRedirect); + try { + new URL(baseRedirect); + } catch (e) { + return Constants.TWITTER_ROOT; + } + return baseRedirect.endsWith('/') ? baseRedirect.slice(0, -1) : baseRedirect; + } + + return Constants.TWITTER_ROOT; +}; + +/* Workaround for some dumb maybe-build time issue where statusRequest isn't ready or something because none of these trigger*/ +const tweetRequest = async (c: Context) => await statusRequest(c); +const _profileRequest = async (c: Context) => await profileRequest(c); + +twitter.get('/:handle{[0-9a-zA-Z_]+}/:endpoint{status(es)?}/:id{[0-9]+}/:language{[a-z]+}?', tweetRequest); +twitter.get('/:prefix{(dir|dl)}/:handle{[0-9a-zA-Z_]+}/status/:id{[0-9]+}/:language{[a-z]+}?', tweetRequest); +twitter.get( + '/:handle{[0-9a-zA-Z_]+}/status/:id{[0-9]+}/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language{[a-z]+}?', + tweetRequest +); +twitter.get( + '/:prefix{(dir|dl)}/:handle{[0-9a-zA-Z_]+}/status/:id{[0-9]+}/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language{[a-z]+}?', + tweetRequest +); + +twitter.get('/version', versionRoute); +twitter.get('/set_base_redirect', setRedirectRequest); +twitter.get('/oembed', oembed); + +twitter.get('/robots.txt', async c => c.text(Strings.ROBOTS_TXT)); + +twitter.get('/i/events/:id', genericTwitterRedirect); +twitter.get('/hashtag/:hashtag', genericTwitterRedirect); + +twitter.get('/:handle', _profileRequest); + +twitter.all('*', async c => c.redirect(Constants.REDIRECT_URL, 302)); diff --git a/src/realms/twitter/routes/oembed.ts b/src/realms/twitter/routes/oembed.ts new file mode 100644 index 0000000..3285d42 --- /dev/null +++ b/src/realms/twitter/routes/oembed.ts @@ -0,0 +1,34 @@ +import { Context } from 'hono'; +import motd from '../../../../motd.json'; +import { Constants } from '../../../constants'; +import { Strings } from '../../../strings'; + +/* Yes, I actually made the endpoint /owoembed. Deal with it. */ +export const oembed = async (c: Context) => { + console.log('oembed hit!'); + const { searchParams } = new URL(c.req.url); + + /* Fallbacks */ + const text = searchParams.get('text') || 'Twitter'; + const author = searchParams.get('author') || 'jack'; + const status = searchParams.get('status') || '20'; + + const random = Math.floor(Math.random() * Object.keys(motd).length); + const [name, url] = Object.entries(motd)[random]; + + const test = { + author_name: text, + author_url: `${Constants.TWITTER_ROOT}/${encodeURIComponent(author)}/status/${status}`, + /* Change provider name if tweet is on deprecated domain. */ + provider_name: + searchParams.get('deprecated') === 'true' ? Strings.DEPRECATED_DOMAIN_NOTICE_DISCORD : name, + provider_url: url, + title: Strings.DEFAULT_AUTHOR_TEXT, + type: 'link', + version: '1.0' + }; + c.header('content-type', 'application/json'); + c.status(200); + /* Stringify and send it on its way! */ + return c.text(JSON.stringify(test)); +}; diff --git a/src/realms/twitter/routes/profile.ts b/src/realms/twitter/routes/profile.ts new file mode 100644 index 0000000..9394d85 --- /dev/null +++ b/src/realms/twitter/routes/profile.ts @@ -0,0 +1,70 @@ +import { Context } from 'hono'; +import { Constants } from '../../../constants'; +import { handleProfile } from '../../../user'; +import { getBaseRedirectUrl } from '../router'; + +/* Handler for User Profiles */ +export const profileRequest = async (c: Context) => { + const handle = c.req.param('handle'); + const url = new URL(c.req.url); + const userAgent = c.req.header('User-Agent') || ''; + const flags: InputFlags = {}; + + /* User Agent matching for embed generators, bots, crawlers, and other automated + tools. It's pretty all-encompassing. Note that Firefox/92 is in here because + Discord sometimes uses the following UA: + + Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 + + I'm not sure why that specific one, it's pretty weird, but this edge case ensures + stuff keeps working. + + On the very rare off chance someone happens to be using specifically Firefox 92, + the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ + const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null; + + /* If not a valid screen name, we redirect to project GitHub */ + if (handle.match(/\w{1,15}/gi)?.[0] !== handle) { + return c.redirect(Constants.REDIRECT_URL, 302); + } + const username = handle.match(/\w{1,15}/gi)?.[0] as string; + /* Check if request is to api.fxtwitter.com */ + if (Constants.API_HOST_LIST.includes(url.hostname)) { + console.log('JSON API request'); + flags.api = true; + } + + const baseUrl = getBaseRedirectUrl(c); + + /* Do not cache if using a custom redirect */ + const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; + + if (cacheControl) { + c.header('cache-control', cacheControl); + } + /* Direct media or API access bypasses bot check, returning same response regardless of UA */ + if (isBotUA || flags.api) { + if (isBotUA) { + console.log(`Matched bot UA ${userAgent}`); + } else { + console.log('Bypass bot check'); + } + + /* This throws the necessary data to handleStatus (in status.ts) */ + const profileResponse = await handleProfile(c, username, flags); + /* Check for custom redirect */ + + if (!isBotUA) { + return c.redirect(`${baseUrl}/${handle}`, 302); + } + + /* Return the response containing embed information */ + return profileResponse; + } else { + /* A human has clicked a fxtwitter.com/:screen_name link! + Obviously we just need to redirect to the user directly.*/ + console.log('Matched human UA', userAgent); + + return c.redirect(`${baseUrl}/${handle}`, 302); + } +}; diff --git a/src/realms/twitter/routes/redirects.ts b/src/realms/twitter/routes/redirects.ts new file mode 100644 index 0000000..cfff1f2 --- /dev/null +++ b/src/realms/twitter/routes/redirects.ts @@ -0,0 +1,98 @@ +import { Context } from 'hono'; +import { Strings } from '../../../strings'; +import { sanitizeText } from '../../../helpers/utils'; +import { getBaseRedirectUrl } from '../router'; +import { Constants } from '../../../constants'; + +export const genericTwitterRedirect = async (c: Context) => { + const url = new URL(c.req.url); + const baseUrl = getBaseRedirectUrl(c); + /* Do not cache if using a custom redirect */ + const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; + + if (cacheControl) { + c.header('cache-control', cacheControl); + } + + return c.redirect(`${baseUrl}${url.pathname}`, 302); +}; + +export const setRedirectRequest = async (c: Context) => { + /* Query params */ + const { searchParams } = new URL(c.req.url); + let url = searchParams.get('url'); + + /* Check that origin either does not exist or is in our domain list */ + const origin = c.req.header('origin'); + if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) { + c.status(403); + + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` + }) + ); + } + + if (!url) { + /* Remove redirect URL */ + c.header( + // eslint-disable-next-line sonarjs/no-duplicate-string + 'set-cookie', + `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly` + ); + c.header( + // eslint-disable-next-line sonarjs/no-duplicate-string + 'content-security-policy', + `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` + ); + c.status(200); + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` + }) + ); + } + + try { + new URL(url); + } catch (e) { + try { + new URL(`https://${url}`); + } catch (e) { + /* URL is not well-formed, remove */ + console.log('Invalid base redirect URL, removing cookie before redirect'); + + c.header( + 'set-cookie', + `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly` + ); + c.header( + 'content-security-policy', + `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` + ); + c.status(200); + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` + }) + ); + } + + url = `https://${url}`; + } + + c.header('set-cookie', `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`); + c.header( + 'content-security-policy', + `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};` + ); + + return c.html( + Strings.MESSAGE_HTML.format({ + message: `Successfully set base redirect, you will now be redirected to ${sanitizeText( + url + )} rather than ${Constants.TWITTER_ROOT}` + }) + ); +}; diff --git a/src/realms/twitter/routes/status.ts b/src/realms/twitter/routes/status.ts new file mode 100644 index 0000000..4b4e354 --- /dev/null +++ b/src/realms/twitter/routes/status.ts @@ -0,0 +1,143 @@ +import { Context } from 'hono'; +import { Constants } from '../../../constants'; +import { getBaseRedirectUrl } from '../router'; +import { handleStatus } from '../../../embed/status'; +import { Strings } from '../../../strings'; + +/* Handler for status (Tweet) request */ +export const statusRequest = async (c: Context) => { + const { prefix, handle, id, mediaNumber, language } = c.req.param(); + console.log('req', JSON.stringify(c.req)) + const url = new URL(c.req.url); + const flags: InputFlags = {}; + + // eslint-disable-next-line sonarjs/no-duplicate-string + const userAgent = c.req.header('User-Agent') || ''; + + /* Let's return our HTML version for wayback machine (we can add other archivers too in future) */ + if ( + ['archive.org', 'Wayback Machine'].some(service => c.req.header('Via')?.includes?.(service)) + ) { + console.log('Request from archive.org'); + flags.archive = true; + } + + /* User Agent matching for embed generators, bots, crawlers, and other automated + tools. It's pretty all-encompassing. Note that Firefox/92 is in here because + Discord sometimes uses the following UA: + + Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 + + I'm not sure why that specific one, it's pretty weird, but this edge case ensures + stuff keeps working. + + On the very rare off chance someone happens to be using specifically Firefox 92, + the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ + const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null || flags?.archive; + + /* Check if domain is a direct media domain (i.e. d.fxtwitter.com), + the tweet is prefixed with /dl/ or /dir/ (for TwitFix interop), or the + tweet ends in .mp4, .jpg, .jpeg, or .png + + Note that .png is not documented because images always redirect to a jpg, + but it will help someone who does it mistakenly on something like Discord + + Also note that all we're doing here is setting the direct flag. If someone + links a video and ends it with .jpg, it will still redirect to a .mp4! */ + if (url.pathname.match(/\/status(es)?\/\d{2,20}\.(mp4|png|jpe?g)/g)) { + console.log('Direct media request by extension'); + flags.direct = true; + } else if (Constants.DIRECT_MEDIA_DOMAINS.includes(url.hostname)) { + console.log('Direct media request by domain'); + flags.direct = true; + } else if (Constants.TEXT_ONLY_DOMAINS.includes(url.hostname)) { + console.log('Text-only embed request'); + flags.textOnly = true; + } else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) { + console.log('Forced instant view request'); + flags.forceInstantView = true; + } else if (prefix === 'dl' || prefix === 'dir') { + console.log('Direct media request by path prefix'); + flags.direct = true; + } + + /* The pxtwitter.com domain is deprecated and Tweets posted after deprecation + date will have a notice saying we've moved to fxtwitter.com! */ + if ( + Constants.DEPRECATED_DOMAIN_LIST.includes(url.hostname) && + BigInt(id?.match(/\d{2,20}/g)?.[0] || 0) > Constants.DEPRECATED_DOMAIN_EPOCH + ) { + console.log('Request to deprecated domain'); + flags.deprecated = true; + } + + /* TODO: Figure out what we're doing with FixTweet / FixupX branding in future */ + if (/fixup/g.test(url.href)) { + console.log(`We're using x domain`); + flags.isXDomain = true; + } else { + console.log(`We're using twitter domain`); + } + + const baseUrl = getBaseRedirectUrl(c); + + if ( + Constants.API_HOST_LIST.includes(url.hostname) + ) { + console.log('JSON API request'); + flags.api = true; + } + + /* Direct media or API access bypasses bot check, returning same response regardless of UA */ + if (isBotUA || flags.direct || flags.api) { + if (isBotUA) { + console.log(`Matched bot UA ${userAgent}`); + } else { + console.log('Bypass bot check'); + } + + /* This throws the necessary data to handleStatus (in status.ts) */ + const statusResponse = await handleStatus( + c, + id?.match(/\d{2,20}/)?.[0] || '0', + mediaNumber ? parseInt(mediaNumber) : undefined, + userAgent, + flags, + language + ); + /* Do not cache if using a custom redirect */ + const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; + + if (cacheControl) { + // eslint-disable-next-line sonarjs/no-duplicate-string + c.header('cache-control', cacheControl); + } + + if (statusResponse) { + /* We're checking if the User Agent is a bot again specifically in case they requested + direct media (d.fxtwitter.com, .mp4/.jpg, etc) but the Tweet contains no media. + + Since we obviously have no media to give the user, we'll just redirect to the Tweet. + Embeds will return as usual to bots as if direct media was never specified. */ + if (!isBotUA && !flags.api) { + const baseUrl = getBaseRedirectUrl(c); + + return c.redirect(`${baseUrl}/${handle || 'i'}/status/${id}`, 302); + } + + c.status(200); + /* Return the response containing embed information */ + return statusResponse; + } else { + /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ + c.status(500); + return c.text(Strings.ERROR_UNKNOWN); + } + } else { + /* A human has clicked a fxtwitter.com/:screen_name/status/:id link! + Obviously we just need to redirect to the Tweet directly.*/ + console.log('Matched human UA', userAgent); + + return c.redirect(`${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, 302); + } +}; diff --git a/src/strings.ts b/src/strings.ts index 3484b95..a90b9d4 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -203,20 +203,6 @@ This is caused by Twitter API downtime or a new bug. Try again in a little while # Check out the docs at https://${API_HOST_LIST.split(',')[0]} to learn how to use it -# ________________ -# / /| -# / / | -# /_______________/ | -# | ___________ | / | -# | | | | / | -# | | | | / | -# | | gaming | | / | -# | |__________| | / | -# | | / | -# | _____ | / | -# | _____________ | / -# |_____________| / - # Good luck, have fun and try not to take over the world! # Instructions below are for robots only, beep boop @@ -242,5 +228,7 @@ Disallow: /owoembed/ Allow: /watch?v=dQw4w9WgXcQ # 0100011101101111011011110110010000100000011000100110111101110100`, - X_DOMAIN_NOTICE: 'FixTweet - 🆕 x.com link? Try fixupx.com' + ROBOTS_TXT_API: `# Crawlers should not crawl API endpoints +User-agent: * +Disallow: /` }; diff --git a/src/types/env.d.ts b/src/types/env.d.ts index c73b538..1e4051b 100644 --- a/src/types/env.d.ts +++ b/src/types/env.d.ts @@ -12,9 +12,4 @@ declare const MOSAIC_DOMAIN_LIST: string; declare const API_HOST_LIST: string; declare const SENTRY_DSN: string; -declare const RELEASE_NAME: string; - -declare const TEST: boolean | undefined; - -declare const TwitterProxy: Fetcher; -declare const AnalyticsEngine: AnalyticsEngineDataset; +declare const RELEASE_NAME: string; \ No newline at end of file diff --git a/src/types/types.d.ts b/src/types/types.d.ts index b3465e6..e5eb0d2 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -37,12 +37,6 @@ interface RenderProperties { flags?: InputFlags; } -interface Request { - params: { - [param: string]: string; - }; -} - interface TweetAPIResponse { code: number; message: string; diff --git a/src/user.ts b/src/user.ts index bf976cb..256ead3 100644 --- a/src/user.ts +++ b/src/user.ts @@ -1,50 +1,50 @@ +import { Context } from 'hono'; import { Constants } from './constants'; import { Strings } from './strings'; import { userAPI } from './providers/twitter/profile'; -export const returnError = (error: string): StatusResponse => { - return { - text: Strings.BASE_HTML.format({ +export const returnError = (c: Context, error: string): Response => { + return c.html( + Strings.BASE_HTML.format({ lang: '', headers: [ ``, `` ].join('') }) - }; + ); }; /* Handler for Twitter users */ export const handleProfile = async ( + c: Context, username: string, - userAgent?: string, - flags?: InputFlags, - event?: FetchEvent -): Promise => { + flags: InputFlags +): Promise => { console.log('Direct?', flags?.direct); - const api = await userAPI(username, event as FetchEvent); + const api = await userAPI(username, c); const user = api?.user as APIUser; /* Catch this request if it's an API response */ // For now we just always return the API response while testing if (flags?.api) { - return { - response: new Response(JSON.stringify(api), { - headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }, - status: api.code - }) - }; + c.status(api.code); + // Add every header from Constants.API_RESPONSE_HEADERS + for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) { + c.header(header, value); + } + return c.json(api); } /* If there was any errors fetching the User, we'll return it */ switch (api.code) { case 401: - return returnError(Strings.ERROR_PRIVATE); + return returnError(c, Strings.ERROR_PRIVATE); case 404: - return returnError(Strings.ERROR_USER_NOT_FOUND); + return returnError(c, Strings.ERROR_USER_NOT_FOUND); case 500: - return returnError(Strings.ERROR_API_FAIL); + return returnError(c, Strings.ERROR_API_FAIL); } /* Base headers included in all responses */ @@ -53,11 +53,10 @@ export const handleProfile = async ( // TODO Add card creation logic here /* Finally, after all that work we return the response HTML! */ - return { - text: Strings.BASE_HTML.format({ + return c.html( + Strings.BASE_HTML.format({ lang: `lang="en"`, headers: headers.join('') - }), - cacheControl: null - }; + }) + ); }; diff --git a/src/worker.ts b/src/worker.ts index 95984dd..f679080 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,664 +1,123 @@ -/* eslint-disable no-case-declarations */ -import { Toucan } from 'toucan-js'; +import { Env, Hono } from 'hono'; +import { timing } from 'hono/timing'; +import { logger } from 'hono/logger'; import { RewriteFrames } from '@sentry/integrations'; - -import { IRequest, Router } from 'itty-router'; -import { Constants } from './constants'; -import { handleStatus } from './embed/status'; +import { sentry } from '@hono/sentry'; import { Strings } from './strings'; +import { Constants } from './constants'; +import { api } from './realms/api/router'; +import { twitter } from './realms/twitter/router'; +import { cacheMiddleware } from './caches'; -import motd from '../motd.json'; -import { sanitizeText } from './helpers/utils'; -import { handleProfile } from './user'; -// import { threadAPIProvider } from './providers/twitter/conversation'; +const noCache = 'max-age=0, no-cache, no-store, must-revalidate'; -declare const globalThis: { - fetchCompletedTime: number; -}; +export const app = new Hono<{ + Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: AnalyticsEngineDataset }; +}>({ + getPath: req => { + let url: URL; -const router = Router(); - -const getBaseRedirectUrl = (request: IRequest) => { - const baseRedirect = request.headers - ?.get('cookie') - ?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0]; - - if (baseRedirect) { - console.log('Found base redirect', baseRedirect); try { - new URL(baseRedirect); - } catch (e) { - return Constants.TWITTER_ROOT; + url = new URL(req.url); + } catch (err) { + return '/error'; } - return baseRedirect.endsWith('/') ? baseRedirect.slice(0, -1) : baseRedirect; - } - - return Constants.TWITTER_ROOT; -}; - -/* Handler for status (Tweet) request */ -const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputFlags = {}) => { - const { handle, id, mediaNumber, language, prefix } = request.params; - const url = new URL(request.url); - // eslint-disable-next-line sonarjs/no-duplicate-string - const userAgent = request.headers.get('User-Agent') || ''; - - /* Let's return our HTML version for wayback machine (we can add other archivers too in future) */ - if ( - ['archive.org', 'Wayback Machine'].some( - service => request.headers.get('Via')?.includes?.(service) - ) - ) { - console.log('Request from archive.org'); - flags.archive = true; - } - - /* User Agent matching for embed generators, bots, crawlers, and other automated - tools. It's pretty all-encompassing. Note that Firefox/92 is in here because - Discord sometimes uses the following UA: - - Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 - - I'm not sure why that specific one, it's pretty weird, but this edge case ensures - stuff keeps working. - - On the very rare off chance someone happens to be using specifically Firefox 92, - the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ - const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null || flags?.archive; - - /* Check if domain is a direct media domain (i.e. d.fxtwitter.com), - the tweet is prefixed with /dl/ or /dir/ (for TwitFix interop), or the - tweet ends in .mp4, .jpg, .jpeg, or .png - - Note that .png is not documented because images always redirect to a jpg, - but it will help someone who does it mistakenly on something like Discord - - Also note that all we're doing here is setting the direct flag. If someone - links a video and ends it with .jpg, it will still redirect to a .mp4! */ - if (url.pathname.match(/\/status(es)?\/\d{2,20}\.(mp4|png|jpe?g)/g)) { - console.log('Direct media request by extension'); - flags.direct = true; - } else if (Constants.DIRECT_MEDIA_DOMAINS.includes(url.hostname)) { - console.log('Direct media request by domain'); - flags.direct = true; - } else if (Constants.TEXT_ONLY_DOMAINS.includes(url.hostname)) { - console.log('Text-only embed request'); - flags.textOnly = true; - } else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) { - console.log('Forced instant view request'); - flags.forceInstantView = true; - } else if (prefix === 'dl' || prefix === 'dir') { - console.log('Direct media request by path prefix'); - flags.direct = true; - } - - /* The pxtwitter.com domain is deprecated and Tweets posted after deprecation - date will have a notice saying we've moved to fxtwitter.com! */ - if ( - Constants.DEPRECATED_DOMAIN_LIST.includes(url.hostname) && - BigInt(id?.match(/\d{2,20}/g)?.[0] || 0) > Constants.DEPRECATED_DOMAIN_EPOCH - ) { - console.log('Request to deprecated domain'); - flags.deprecated = true; - } - - /* TODO: Figure out what we're doing with FixTweet / FixupX branding in future */ - if (/fixup/g.test(url.href)) { - console.log(`We're using x domain`); - flags.isXDomain = true; - } else { - console.log(`We're using twitter domain`); - } - - const baseUrl = getBaseRedirectUrl(request); - - /* Check if request is to api.fxtwitter.com, or the tweet is appended with .json - Note that unlike TwitFix, FixTweet will never generate embeds for .json, and - in fact we only support .json because it's what people using TwitFix API would - be used to. */ - if ( - url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null || - Constants.API_HOST_LIST.includes(url.hostname) - ) { - console.log('JSON API request'); - flags.api = true; - } - - /* Direct media or API access bypasses bot check, returning same response regardless of UA */ - if (isBotUA || flags.direct || flags.api) { - if (isBotUA) { - console.log(`Matched bot UA ${userAgent}`); + const baseHostName = url.hostname.split('.').slice(-2).join('.'); + let realm = 'twitter'; + /* Override if in API_HOST_LIST. Note that we have to check full hostname for this. */ + if (Constants.API_HOST_LIST.includes(url.hostname)) { + realm = 'api'; + console.log('API realm'); + } else if (Constants.STANDARD_DOMAIN_LIST.includes(baseHostName)) { + console.log(); + realm = 'twitter'; + console.log('Twitter realm'); } else { - console.log('Bypass bot check'); + console.log(`Domain not assigned to realm, falling back to Twitter: ${url.hostname}`); } + /* Defaults to Twitter realm if unknown domain specified (such as the *.workers.dev hostname or deprecated domain) */ - /* This throws the necessary data to handleStatus (in status.ts) */ - const statusResponse = await handleStatus( - id?.match(/\d{2,20}/)?.[0] || '0', - mediaNumber ? parseInt(mediaNumber) : undefined, - userAgent, - flags, - language, - event, - request - ); - - /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */ - if (statusResponse.response) { - console.log('handleStatus sent response'); - return statusResponse.response; - } else if (statusResponse.text) { - console.log('handleStatus sent embed'); - /* We're checking if the User Agent is a bot again specifically in case they requested - direct media (d.fxtwitter.com, .mp4/.jpg, etc) but the Tweet contains no media. - - Since we obviously have no media to give the user, we'll just redirect to the Tweet. - Embeds will return as usual to bots as if direct media was never specified. */ - if (!isBotUA && !flags.api) { - const baseUrl = getBaseRedirectUrl(request); - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle || 'i'}/status/${id}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); - } - - let headers = Constants.RESPONSE_HEADERS; - - if (statusResponse.cacheControl) { - headers = { - ...headers, - 'cache-control': - baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : statusResponse.cacheControl - }; - } - - /* Return the response containing embed information */ - return new Response(statusResponse.text, { - headers: headers, - status: 200 - }); - } else { - /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ - return new Response(Strings.ERROR_UNKNOWN, { - headers: Constants.RESPONSE_HEADERS, - status: 500 - }); - } - } else { - globalThis.fetchCompletedTime = performance.now(); - /* A human has clicked a fxtwitter.com/:screen_name/status/:id link! - Obviously we just need to redirect to the Tweet directly.*/ - console.log('Matched human UA', userAgent); - - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); + console.log(`/${realm}${url.pathname}`); + return `/${realm}${url.pathname}`; } -}; - -/* Handler for User Profiles */ -const profileRequest = async (request: IRequest, event: FetchEvent, flags: InputFlags = {}) => { - const { handle } = request.params; - const url = new URL(request.url); - const userAgent = request.headers.get('User-Agent') || ''; - - /* User Agent matching for embed generators, bots, crawlers, and other automated - tools. It's pretty all-encompassing. Note that Firefox/92 is in here because - Discord sometimes uses the following UA: - - Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0 - - I'm not sure why that specific one, it's pretty weird, but this edge case ensures - stuff keeps working. - - On the very rare off chance someone happens to be using specifically Firefox 92, - the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */ - const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null; - - /* If not a valid screen name, we redirect to project GitHub */ - if (handle.match(/\w{1,15}/gi)?.[0] !== handle) { - return Response.redirect(Constants.REDIRECT_URL, 302); - } - const username = handle.match(/\w{1,15}/gi)?.[0] as string; - /* Check if request is to api.fxtwitter.com */ - if (Constants.API_HOST_LIST.includes(url.hostname)) { - console.log('JSON API request'); - flags.api = true; - } - - /* Direct media or API access bypasses bot check, returning same response regardless of UA */ - if (isBotUA || flags.api) { - if (isBotUA) { - console.log(`Matched bot UA ${userAgent}`); - } else { - console.log('Bypass bot check'); - } - - /* This throws the necessary data to handleStatus (in status.ts) */ - const profileResponse = await handleProfile(username, userAgent, flags, event); - - /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */ - if (profileResponse.response) { - console.log('handleProfile sent response'); - return profileResponse.response; - } else if (profileResponse.text) { - console.log('handleProfile sent embed'); - /* TODO This check has purpose in the original handleStatus handler, but I'm not sure if this edge case can happen here */ - const baseUrl = getBaseRedirectUrl(request); - /* Check for custom redirect */ - - if (!isBotUA) { - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); - } - - let headers = Constants.RESPONSE_HEADERS; - - if (profileResponse.cacheControl) { - headers = { - ...headers, - 'cache-control': - baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : profileResponse.cacheControl - }; - } - - /* Return the response containing embed information */ - return new Response(profileResponse.text, { - headers: headers, - status: 200 - }); - } else { - /* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */ - return new Response(Strings.ERROR_UNKNOWN, { - headers: { ...Constants.RESPONSE_HEADERS, 'cache-control': 'max-age=0' }, - status: 500 - }); - } - } else { - /* A human has clicked a fxtwitter.com/:screen_name link! - Obviously we just need to redirect to the user directly.*/ - console.log('Matched human UA', userAgent); - - const baseUrl = getBaseRedirectUrl(request); - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}/${handle}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); - } -}; - -const genericTwitterRedirect = async (request: IRequest) => { - const url = new URL(request.url); - const baseUrl = getBaseRedirectUrl(request); - /* Do not cache if using a custom redirect */ - const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined; - - return new Response(null, { - status: 302, - headers: { - Location: `${baseUrl}${url.pathname}`, - ...(cacheControl ? { 'cache-control': cacheControl } : {}) - } - }); -}; - -const versionRequest = async (request: IRequest) => { - globalThis.fetchCompletedTime = performance.now(); - return new Response( - Strings.VERSION_HTML.format({ - rtt: request.cf?.clientTcpRtt ? `🏓 ${request.cf.clientTcpRtt} ms RTT` : '', - colo: (request.cf?.colo as string) ?? '??', - httpversion: (request.cf?.httpProtocol as string) ?? 'Unknown HTTP Version', - tlsversion: (request.cf?.tlsVersion as string) ?? 'Unknown TLS Version', - ip: - request.headers.get('x-real-ip') ?? request.headers.get('cf-connecting-ip') ?? 'Unknown IP', - city: (request.cf?.city as string) ?? 'Unknown City', - region: (request.cf?.region as string) ?? request.cf?.country ?? 'Unknown Region', - country: (request.cf?.country as string) ?? 'Unknown Country', - asn: `AS${request.cf?.asn ?? '??'} (${request.cf?.asOrganization ?? 'Unknown ASN'})`, - ua: sanitizeText(request.headers.get('user-agent') ?? 'Unknown User Agent') - }), - { - headers: { - ...Constants.RESPONSE_HEADERS, - 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate' - }, - status: 200 - } - ); -}; - -const setRedirectRequest = async (request: IRequest) => { - /* Query params */ - const { searchParams } = new URL(request.url); - let url = searchParams.get('url'); - - /* Check that origin either does not exist or is in our domain list */ - const origin = request.headers.get('origin'); - if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) { - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.` - }), - { - headers: Constants.RESPONSE_HEADERS, - status: 403 - } - ); - } - - if (!url) { - /* Remove redirect URL */ - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Your base redirect has been cleared. To set one, please pass along the url parameter.` - }), - { - headers: { - 'set-cookie': `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`, - 'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`, - ...Constants.RESPONSE_HEADERS - }, - status: 200 - } - ); - } - - try { - new URL(url); - } catch (e) { - try { - new URL(`https://${url}`); - } catch (e) { - /* URL is not well-formed, remove */ - console.log('Invalid base redirect URL, removing cookie before redirect'); - - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net` - }), - { - headers: { - 'set-cookie': `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`, - 'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join( - ' ' - )};`, - ...Constants.RESPONSE_HEADERS - }, - status: 200 - } - ); - } - - url = `https://${url}`; - } - - /* Set cookie for url */ - return new Response( - Strings.MESSAGE_HTML.format({ - message: `Successfully set base redirect, you will now be redirected to ${sanitizeText( - url - )} rather than ${Constants.TWITTER_ROOT}` - }), - { - headers: { - 'set-cookie': `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`, - 'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`, - ...Constants.RESPONSE_HEADERS - }, - status: 200 - } - ); -}; - -/* TODO: is there any way to consolidate these stupid routes for itty-router? - I couldn't find documentation allowing for regex matching */ -router.get('/:prefix?/:handle/status/:id', statusRequest); -router.get('/:prefix?/:handle/status/:id/photo/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/photos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/video/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/videos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/photo/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/photos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/video/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/videos/:mediaNumber', statusRequest); -router.get('/:prefix?/:handle/status/:id/:language', statusRequest); -router.get('/:prefix?/:handle/statuses/:id/:language', statusRequest); -router.get('/status/:id', statusRequest); -router.get('/status/:id/:language', statusRequest); -router.get('/version', versionRequest); -router.get('/set_base_redirect', setRedirectRequest); -// router.get('/v2/twitter/thread/:id', threadAPIProvider) - -/* Oembeds (used by Discord to enhance responses) - -Yes, I actually made the endpoint /owoembed. Deal with it. */ -router.get('/owoembed', async (request: IRequest) => { - globalThis.fetchCompletedTime = performance.now(); - console.log('oembed hit!'); - const { searchParams } = new URL(request.url); - - /* Fallbacks */ - const text = searchParams.get('text') || 'Twitter'; - const author = searchParams.get('author') || 'jack'; - const status = searchParams.get('status') || '20'; - - const random = Math.floor(Math.random() * Object.keys(motd).length); - const [name, url] = Object.entries(motd)[random]; - - const test = { - author_name: text, - author_url: `${Constants.TWITTER_ROOT}/${encodeURIComponent(author)}/status/${status}`, - /* Change provider name if tweet is on deprecated domain. */ - provider_name: - searchParams.get('deprecated') === 'true' ? Strings.DEPRECATED_DOMAIN_NOTICE_DISCORD : name, - provider_url: url, - title: Strings.DEFAULT_AUTHOR_TEXT, - type: 'link', - version: '1.0' - }; - /* Stringify and send it on its way! */ - return new Response(JSON.stringify(test), { - headers: { - ...Constants.RESPONSE_HEADERS, - 'content-type': 'application/json' - }, - status: 200 - }); }); -/* Pass through profile requests to Twitter. - We don't currently have custom profile cards yet, - but it's something we might do. Maybe. */ -router.get('/:handle', profileRequest); -router.get('/:handle/', profileRequest); -router.get('/i/events/:id', genericTwitterRedirect); -router.get('/hashtag/:hashtag', genericTwitterRedirect); - -/* If we don't understand the route structure at all, we'll - redirect to GitHub (normal domains) or API docs (api.fxtwitter.com) */ -router.get('*', async (request: IRequest) => { - const url = new URL(request.url); - - if (Constants.API_HOST_LIST.includes(url.hostname)) { - return Response.redirect(Constants.API_DOCS_URL, 302); - } - return Response.redirect(Constants.REDIRECT_URL, 302); -}); - -/* Wrapper to handle caching, and misc things like catching robots.txt */ -export const cacheWrapper = async (request: Request, event?: FetchEvent): Promise => { - const startTime = performance.now(); - const userAgent = request.headers.get('User-Agent') || ''; - // https://developers.cloudflare.com/workers/examples/cache-api/ - const cacheUrl = new URL( - userAgent.includes('Telegram') - ? `${request.url}&telegram` - : userAgent.includes('Discord') - ? `${request.url}&discord` - : request.url - ); - - console.log(`Hello from ⛅ ${request.cf?.colo || 'UNK'}`); - console.log('userAgent', userAgent); - console.log('cacheUrl', cacheUrl); - - const cacheKey = new Request(cacheUrl.toString(), request); - const cache = caches.default; - - /* Itty-router doesn't seem to like routing file names because whatever, - so we just handle it in the caching layer instead. Kinda hacky, but whatever. */ - if (cacheUrl.pathname === '/robots.txt') { - return new Response(Strings.ROBOTS_TXT, { - headers: { - ...Constants.RESPONSE_HEADERS, - 'content-type': 'text/plain' - }, - status: 200 - }); - } - - switch (request.method) { - case 'GET': - if ( - !Constants.API_HOST_LIST.includes(cacheUrl.hostname) && - !request.headers?.get('Cookie')?.includes('base_redirect') - ) { - /* cache may be undefined in tests */ - const cachedResponse = await cache.match(cacheKey); - - if (cachedResponse) { - console.log('Cache hit'); - return cachedResponse; - } - - console.log('Cache miss'); - } - - const response = await router.handle(request, event); - - /* Store the fetched response as cacheKey - Use waitUntil so you can return the response without blocking on - writing to cache */ - try { - event && event.waitUntil(cache.put(cacheKey, response.clone())); - } catch (error) { - console.error((error as Error).stack); - } - - const endTime = performance.now(); - const timeSinceFetch = endTime - (globalThis.fetchCompletedTime || 0); - const timeSinceStart = endTime - startTime; - console.log( - `Request took ${timeSinceStart}ms, of which ${timeSinceFetch}ms was CPU time after last fetch` - ); - - return response; - /* Telegram sends this from Webpage Bot, and Cloudflare sends it if we purge cache, and we respect it. - PURGE is not defined in an RFC, but other servers like Nginx apparently use it. */ - case 'PURGE': - console.log('Purging cache as requested'); - await cache.delete(cacheKey); - return new Response('', { status: 200 }); - /* yes, we do give HEAD */ - case 'HEAD': - return new Response('', { - headers: Constants.RESPONSE_HEADERS, - status: 200 - }); - /* We properly state our OPTIONS when asked */ - case 'OPTIONS': - return new Response('', { - headers: { - allow: Constants.RESPONSE_HEADERS.allow - }, - status: 204 - }); - default: - return new Response('', { status: 405 }); - } -}; - -/* Wrapper around Sentry, used for catching uncaught exceptions */ -const sentryWrapper = async (event: FetchEvent, test = false): Promise => { - let sentry: null | Toucan = null; - - if (typeof SENTRY_DSN !== 'undefined' && SENTRY_DSN && !test) { - /* We use Toucan for Sentry. Toucan is a Sentry SDK designed for Cloudflare Workers / DOs */ - sentry = new Toucan({ +if (SENTRY_DSN) { + app.use( + '*', + sentry({ dsn: SENTRY_DSN, - context: event, - request: event.request, requestDataOptions: { allowedHeaders: /(.*)/, allowedSearchParams: /(.*)/ }, - /* TODO: Figure out what changed between @sentry/integration 7.65.0 and 7.66.0 - https://github.com/getsentry/sentry-javascript/compare/7.65.0...7.66.0 - which caused types to go apeshit */ // eslint-disable-next-line @typescript-eslint/no-explicit-any integrations: [new RewriteFrames({ root: '/' }) as any], - /* event includes 'waitUntil', which is essential for Sentry logs to be delivered. - Also includes 'request' -- no need to set it separately. */ release: RELEASE_NAME - }); - } - - /* Responds with either a returned response (good!!!) or returns - a crash response (bad!!!) */ - event.respondWith( - (async (): Promise => { - try { - return await cacheWrapper(event.request, event); - } catch (err: unknown) { - sentry && sentry.captureException(err); - - /* workaround for silly TypeScript things */ - const error = err as Error; - console.error(error.stack); - - return new Response(Strings.ERROR_HTML, { - headers: { - ...Constants.RESPONSE_HEADERS, - 'content-type': 'text/html', - 'cache-control': 'max-age=0, no-cache, no-store, must-revalidate' - }, - status: 200 - }); - } - })() + }) ); +} + +app.use('*', async (c, next) => { + /* Apply all headers from Constants.RESPONSE_HEADERS */ + for (const [header, value] of Object.entries(Constants.RESPONSE_HEADERS)) { + c.header(header, value); + } + await next(); +}); + +app.onError((err, c) => { + c.get('sentry')?.captureException?.(err); + console.error(err.stack); + c.status(200); + c.header('cache-control', noCache); + + return c.html(Strings.ERROR_HTML); +}); + +const customLogger = (message: string, ...rest: string[]) => { + console.log(message, ...rest); }; -/* Event to receive web requests on Cloudflare Worker */ -addEventListener('fetch', (event: FetchEvent) => { - sentryWrapper(event); +app.use('*', logger(customLogger)); + +app.use('*', async (c, next) => { + if (c.req.raw.cf) { + console.log(`Hello from ⛅ ${c.req.raw.cf.colo ?? 'UNK'}`); + } + console.log('userAgent', c.req.header('user-agent')); + await next(); }); + +app.use('*', cacheMiddleware()); +app.use('*', timing({ enabled: false })); + +app.route(`/api`, api); +app.route(`/twitter`, twitter); + +app.all('/error', async c => { + c.header('cache-control', noCache); + c.status(400); + return c.body(''); +}); + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext) { + try { + return await app.fetch(request, env, ctx); + } catch (err) { + console.error(err); + console.log('Ouch, that error hurt so much Sentry couldnt catch it'); + + return new Response(Strings.ERROR_HTML, { + headers: { + ...Constants.RESPONSE_HEADERS, + 'content-type': 'text/html;charset=utf-8', + 'cache-control': noCache + }, + status: 200 + }); + } + } +}; \ No newline at end of file diff --git a/test/index.test.ts b/test/worker.test.ts similarity index 95% rename from test/index.test.ts rename to test/worker.test.ts index 45185e4..89d0016 100644 --- a/test/index.test.ts +++ b/test/worker.test.ts @@ -1,4 +1,4 @@ -import { cacheWrapper } from '../src/worker'; +import { app } from '../src/worker'; const botHeaders = { 'User-Agent': 'Discordbot/2.0' }; const humanHeaders = { @@ -29,13 +29,13 @@ if (!globalThis.performance.now) { } test('Home page redirect', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://fxtwitter.com', { method: 'GET', headers: botHeaders }) ); - const resultHuman = await cacheWrapper( + const resultHuman = await app.request( new Request('https://fxtwitter.com', { method: 'GET', headers: humanHeaders @@ -48,7 +48,7 @@ test('Home page redirect', async () => { }); test('Tweet redirect human', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: humanHeaders @@ -59,7 +59,7 @@ test('Tweet redirect human', async () => { }); test('Tweet redirect human custom base redirect', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: { @@ -73,7 +73,7 @@ test('Tweet redirect human custom base redirect', async () => { }); test('Twitter moment redirect', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request( 'https://fxtwitter.com/i/events/1572638642127966214?t=0UK7Ny-Jnsp-dUGzlb-M8w&s=35', { @@ -87,7 +87,7 @@ test('Twitter moment redirect', async () => { }); test('Tweet response robot', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://fxtwitter.com/jack/status/20', { method: 'GET', headers: botHeaders @@ -97,7 +97,7 @@ test('Tweet response robot', async () => { }); test('API fetch basic Tweet', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://api.fxtwitter.com/status/20', { method: 'GET', headers: botHeaders @@ -132,7 +132,7 @@ test('API fetch basic Tweet', async () => { }); // test('API fetch video Tweet', async () => { -// const result = await cacheWrapper( +// const result = await app.request( // new Request('https://api.fxtwitter.com/X/status/854416760933556224', { // method: 'GET', // headers: botHeaders @@ -177,7 +177,7 @@ test('API fetch basic Tweet', async () => { // }); // test('API fetch multi-photo Tweet', async () => { -// const result = await cacheWrapper( +// const result = await app.request( // new Request('https://api.fxtwitter.com/Twitter/status/1445094085593866246', { // method: 'GET', // headers: botHeaders @@ -224,7 +224,7 @@ test('API fetch basic Tweet', async () => { // }); // test('API fetch poll Tweet', async () => { -// const result = await cacheWrapper( +// const result = await app.request( // new Request('https://api.fxtwitter.com/status/1055475950543167488', { // method: 'GET', // headers: botHeaders @@ -273,7 +273,7 @@ test('API fetch basic Tweet', async () => { // }); test('API fetch user', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://api.fxtwitter.com/x', { method: 'GET', headers: botHeaders @@ -303,7 +303,7 @@ test('API fetch user', async () => { }); test('API fetch user that does not exist', async () => { - const result = await cacheWrapper( + const result = await app.request( new Request('https://api.fxtwitter.com/usesaahah123', { method: 'GET', headers: botHeaders