From 91878e91c7530e78ba47d5c4a1dba2c9340dc89d Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Sun, 12 Nov 2023 01:49:32 -0500 Subject: [PATCH] Improve reliability with timeouts, improve error codes --- src/fetch.ts | 27 +-- src/helpers/translate.ts | 10 +- src/helpers/utils.ts | 23 +++ src/providers/twitter/conversation.ts | 12 +- src/providers/twitter/profile.ts | 12 +- src/strings.ts | 240 +++++++++++++++----------- src/worker.ts | 33 +++- 7 files changed, 222 insertions(+), 135 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index 4056aab..f0ed0c4 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -2,6 +2,7 @@ import { Context } from 'hono'; import { Constants } from './constants'; import { Experiment, experimentCheck } from './experiments'; import { generateUserAgent } from './helpers/useragent'; +import { withTimeout } from './helpers/utils'; const API_ATTEMPTS = 3; let wasElongatorDisabled = false; @@ -131,24 +132,26 @@ export const twitterFetch = async ( headers['x-guest-token'] = guestToken; let response: unknown; - let apiRequest; + let apiRequest: Response | null = null; try { if (useElongator && typeof c.env?.TwitterProxy !== 'undefined') { console.log('Fetching using elongator'); const performanceStart = performance.now(); - apiRequest = await c.env?.TwitterProxy.fetch(url, { + apiRequest = await withTimeout((signal: AbortSignal) => c.env?.TwitterProxy.fetch(url, { method: 'GET', - headers: headers - }); + headers: headers, + signal: signal + })); const performanceEnd = performance.now(); console.log(`Elongator request successful after ${performanceEnd - performanceStart}ms`); } else { const performanceStart = performance.now(); - apiRequest = await fetch(url, { + apiRequest = await withTimeout((signal: AbortSignal) => fetch(url, { method: 'GET', - headers: headers - }); + headers: headers, + signal: signal + })); const performanceEnd = performance.now(); console.log(`Guest API request successful after ${performanceEnd - performanceStart}ms`); } @@ -159,9 +162,9 @@ export const twitterFetch = async ( It's uncommon, but it happens */ console.error('Unknown error while fetching from API', e); /* Elongator returns strings to communicate downstream errors */ - if (String(e).indexOf('Status not found')) { + if (String(e).indexOf('Status not found') !== -1) { console.log('Tweet was not found'); - return {}; + return null; } try{ !useElongator && @@ -194,7 +197,7 @@ export const twitterFetch = async ( continue; } - const remainingRateLimit = parseInt(apiRequest.headers.get('x-rate-limit-remaining') || '0'); + const remainingRateLimit = parseInt(apiRequest?.headers.get('x-rate-limit-remaining') || '0'); console.log(`Remaining rate limit: ${remainingRateLimit} requests`); /* Running out of requests within our rate limit, let's purge the cache */ if (!useElongator && remainingRateLimit < 10) { @@ -247,7 +250,7 @@ export const twitterFetch = async ( console.log('Twitter has repeatedly denied our requests, so we give up now'); - return {}; + return null; }; export const fetchUser = async ( @@ -287,7 +290,7 @@ export const fetchUser = async ( } return !( response?.data?.user?.result?.__typename !== 'User' || - typeof response.data.user.result.legacy === 'undefined' + typeof response?.data?.user?.result?.legacy === 'undefined' ); /* return !( diff --git a/src/helpers/translate.ts b/src/helpers/translate.ts index e1368e0..c9682f6 100644 --- a/src/helpers/translate.ts +++ b/src/helpers/translate.ts @@ -1,5 +1,6 @@ import { Context } from 'hono'; import { Constants } from '../constants'; +import { withTimeout } from './utils'; /* Handles translating Tweets when asked! */ export const translateTweet = async ( @@ -53,10 +54,11 @@ export const translateTweet = async ( try { const url = `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`; console.log(url, headers); - translationApiResponse = await c.env?.TwitterProxy.fetch(url, { + translationApiResponse = await withTimeout((signal: AbortSignal) => c.env?.TwitterProxy.fetch(url, { method: 'GET', - headers: headers - }); + headers: headers, + signal: signal + })) as Response; translationResults = (await translationApiResponse.json()) as TranslationPartial; console.log(`translationResults`, translationResults); @@ -69,6 +71,6 @@ export const translateTweet = async ( return translationResults; } catch (e: unknown) { console.error('Unknown error while fetching from Translation API', e); - return {} as TranslationPartial; // No work to do + return null; } }; diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index fc04bfd..5607879 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -32,6 +32,29 @@ export const truncateWithEllipsis = (str: string, maxLength: number): string => return truncated.length < str.length ? truncated + '…' : truncated; }; +export async function withTimeout( + asyncTask: (signal: AbortSignal) => Promise, + timeout: number = 3000 +): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeout); + + try { + const result = await asyncTask(controller.signal); + /* Clear the timeout if the task completes in time */ + clearTimeout(timeoutId); + console.log(`Clearing timeout after ${timeout}ms`); + return result; + } catch (error) { + if ((error as Error).name === 'AbortError') { + throw new Error('Asynchronous task was aborted due to timeout'); + } else { + /* Re-throw other errors for further handling */ + throw error as Error; + } + } +} + const numberFormat = new Intl.NumberFormat('en-US'); export const formatNumber = (num: number) => numberFormat.format(num); diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index da6bf9b..9abf022 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -62,15 +62,14 @@ export const fetchTweetDetail = async ( const conversation = _conversation as TweetDetailResult; const tweet = findTweetInBucket( status, - processResponse(conversation.data?.threaded_conversation_with_injections_v2?.instructions) + processResponse(conversation?.data?.threaded_conversation_with_injections_v2?.instructions) ); if (tweet && isGraphQLTweet(tweet)) { return true; } console.log('invalid graphql tweet', conversation); const firstInstruction = ( - conversation.data?.threaded_conversation_with_injections_v2 - .instructions?.[0] as TimelineAddEntriesInstruction + conversation?.data?.threaded_conversation_with_injections_v2?.instructions?.[0] as TimelineAddEntriesInstruction )?.entries?.[0]; if ( ( @@ -297,8 +296,7 @@ export const constructTwitterThread = async ( console.log(response); const firstInstruction = ( - response.data?.threaded_conversation_with_injections_v2 - .instructions?.[0] as TimelineAddEntriesInstruction + response?.data?.threaded_conversation_with_injections_v2?.instructions?.[0] as TimelineAddEntriesInstruction )?.entries?.[0]; if ( ( @@ -338,7 +336,7 @@ export const constructTwitterThread = async ( } const bucket = processResponse( - response.data.threaded_conversation_with_injections_v2.instructions + response?.data?.threaded_conversation_with_injections_v2?.instructions ?? [] ); const originalTweet = findTweetInBucket(id, bucket); @@ -482,7 +480,7 @@ export const constructTwitterThread = async ( break; } const cursorResponse = processResponse( - loadCursor.data.threaded_conversation_with_injections_v2.instructions + loadCursor?.data?.threaded_conversation_with_injections_v2.instructions ); bucket.tweets = cursorResponse.tweets.concat( filterBucketTweets(bucket.tweets, originalTweet) diff --git a/src/providers/twitter/profile.ts b/src/providers/twitter/profile.ts index 0f4ad81..8edd16f 100644 --- a/src/providers/twitter/profile.ts +++ b/src/providers/twitter/profile.ts @@ -70,9 +70,13 @@ export const convertToApiUser = (user: GraphQLUser, legacyAPI = false): APIUser const populateUserProperties = async ( response: GraphQLUserResponse, legacyAPI = false -): Promise => { - const user = response.data.user.result; - return convertToApiUser(user, legacyAPI); +): Promise => { + const user = response?.data?.user?.result; + if (user) { + return convertToApiUser(user, legacyAPI); + } + + return null; }; /* API for Twitter profiles (Users) @@ -95,7 +99,7 @@ export const userAPI = async ( const apiUser: APIUser = (await populateUserProperties(userResponse, true)) as APIUser; /* Currently, we haven't rolled this out as it's part of the proto-v2 API */ - delete apiUser.global_screen_name; + delete apiUser?.global_screen_name; /* Finally, staple the User to the response and return it */ response.user = apiUser; diff --git a/src/strings.ts b/src/strings.ts index a90b9d4..f2dbefa 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -31,16 +31,94 @@ export const Strings = { ███ Worker build ${RELEASE_NAME} -->{headers}{body}`, - ERROR_HTML: ` +ERROR_HTML: ` + + + + + + + :( + + + +

Owie :(

+

You hit a snag that broke ${BRANDING_NAME}. It's not your fault though—This is usually caused by a Twitter outage or a new bug.

+

${RELEASE_NAME}

+ +` + .replace(/( {2})/g, '') + .replace(/>\s+<'), + TIMEOUT_ERROR_HTML: ` + + + + + + + :( + + + +

Gateway Timeout

+

A downstream timeout occurred while trying to generate the embed. Please try again in a little while.

+

${RELEASE_NAME}

+ +` + .replace(/( {2})/g, '') + .replace(/>\s+<'), + VERSION_HTML: ` - + + + + - :( + Edge Connection: + {rtt} 📶 {httpversion} 🔒 {tlsversion} ➡ ⛅ {colo} + " property="og:description"/> + ${BRANDING_NAME} + + +

${BRANDING_NAME}

+

A better way to embed X / Twitter posts on Discord, Telegram, and more.

+

Worker release: ${RELEASE_NAME}

+
+

Stats for nerds:

+

Edge Connection: + {rtt} 📶 {httpversion} 🔒 {tlsversion} ➡ {colo}

+

User Agent: + {ua}

+ + ` + .replace(/( {2})/g, '') + .replace(/>\s+<'), + MESSAGE_HTML: ` + + + + + + + ${BRANDING_NAME} + -

Owie :(

-

You hit a snag that broke ${BRANDING_NAME}. It's not your fault though—This is usually caused by a Twitter outage or a new bug.

-

${RELEASE_NAME}

+

${BRANDING_NAME}

+

{message}

` - .replace(/( {2})/g, '') - .replace(/>\s+<'), - VERSION_HTML: ` - - - - - - - - - - - ${BRANDING_NAME} - - - -

${BRANDING_NAME}

-

A better way to embed X / Twitter posts on Discord, Telegram, and more.

-

Worker release: ${RELEASE_NAME}

-
-

Stats for nerds:

-

Edge Connection: - {rtt} 📶 {httpversion} 🔒 {tlsversion} ➡ {colo}

-

User Agent: - {ua}

- - ` - .replace(/( {2})/g, '') - .replace(/>\s+<'), - MESSAGE_HTML: ` - - - - - - - ${BRANDING_NAME} - - - -

${BRANDING_NAME}

-

{message}

- - ` - .replace(/( {2})/g, '') - .replace(/>\s+<'), + .replace(/( {2})/g, '') + .replace(/>\s+<'), DEFAULT_AUTHOR_TEXT: 'Twitter', QUOTE_TEXT: `↘️ Quoting {name} (@{screen_name})`, diff --git a/src/worker.ts b/src/worker.ts index 0edc058..bda68c1 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -8,6 +8,7 @@ import { Constants } from './constants'; import { api } from './realms/api/router'; import { twitter } from './realms/twitter/router'; import { cacheMiddleware } from './caches'; +import { withTimeout } from './helpers/utils'; const noCache = 'max-age=0, no-cache, no-store, must-revalidate'; @@ -79,7 +80,15 @@ app.use('*', async (c, next) => { app.onError((err, c) => { c.get('sentry')?.captureException?.(err); console.error(err.stack); - c.status(200); + let errorCode = 500; + if (err.name === 'AbortError') { + errorCode = 504; + } + /* We return it as a 200 so embedded applications can display the error */ + if (c.req.header('User-Agent')?.match(/(discordbot|telegrambot|facebook|whatsapp|firefox\/92|vkshare)/gi)) { + errorCode = 200; + } + c.status(errorCode); c.header('cache-control', noCache); return c.html(Strings.ERROR_HTML); @@ -107,25 +116,41 @@ app.route(`/twitter`, twitter); app.all('/error', async c => { c.header('cache-control', noCache); + + if (c.req.header('User-Agent')?.match(/(discordbot|telegrambot|facebook|whatsapp|firefox\/92|vkshare)/gi)) { + c.status(200); + return c.html(Strings.ERROR_HTML); + } c.status(400); + /* We return it as a 200 so embedded applications can display the error */ return c.body(''); }); export default { async fetch(request: Request, env: Env, ctx: ExecutionContext) { try { - return await app.fetch(request, env, ctx); + return await withTimeout(async () => app.fetch(request, env, ctx), 10); } catch (err) { console.error(err); + const e = err as Error; console.log(`Ouch, that error hurt so much Sentry couldn't catch it`); + console.log(e.stack); + let errorCode = 500; + if (e.name === 'AbortError') { + errorCode = 504; + } + /* We return it as a 200 so embedded applications can display the error */ + if (request.headers.get('user-agent')?.match(/(discordbot|telegrambot|facebook|whatsapp|firefox\/92|vkshare)/gi)) { + errorCode = 200; + } - return new Response(Strings.ERROR_HTML, { + return new Response(e.name === 'AbortError' ? Strings.TIMEOUT_ERROR_HTML : Strings.ERROR_HTML, { headers: { ...Constants.RESPONSE_HEADERS, 'content-type': 'text/html;charset=utf-8', 'cache-control': noCache }, - status: 200 + status: errorCode }); } }