Improve reliability with timeouts, improve error codes

src/fetch.ts

@@ -2,6 +2,7 @@ import { Context } from 'hono';
 import { Constants } from './constants';
 import { Experiment, experimentCheck } from './experiments';
 import { generateUserAgent } from './helpers/useragent';
+import { withTimeout } from './helpers/utils';
 
 const API_ATTEMPTS = 3;
 let wasElongatorDisabled = false;
@@ -131,24 +132,26 @@ export const twitterFetch = async (
     headers['x-guest-token'] = guestToken;
 
     let response: unknown;
-    let apiRequest;
+    let apiRequest: Response | null = null;
 
     try {
       if (useElongator && typeof c.env?.TwitterProxy !== 'undefined') {
         console.log('Fetching using elongator');
         const performanceStart = performance.now();
-        apiRequest = await c.env?.TwitterProxy.fetch(url, {
+        apiRequest = await withTimeout((signal: AbortSignal) => c.env?.TwitterProxy.fetch(url, {
           method: 'GET',
-          headers: headers
-        });
+          headers: headers,
+          signal: signal
+        }));
         const performanceEnd = performance.now();
         console.log(`Elongator request successful after ${performanceEnd - performanceStart}ms`);
       } else {
         const performanceStart = performance.now();
-        apiRequest = await fetch(url, {
+        apiRequest = await withTimeout((signal: AbortSignal) => fetch(url, {
           method: 'GET',
-          headers: headers
-        });
+          headers: headers,
+          signal: signal
+        }));
         const performanceEnd = performance.now();
         console.log(`Guest API request successful after ${performanceEnd - performanceStart}ms`);
       }
@@ -159,9 +162,9 @@ export const twitterFetch = async (
          It's uncommon, but it happens */
       console.error('Unknown error while fetching from API', e);
       /* Elongator returns strings to communicate downstream errors */
-      if (String(e).indexOf('Status not found')) {
+      if (String(e).indexOf('Status not found') !== -1) {
         console.log('Tweet was not found');
-        return {};
+        return null;
       }
       try{
         !useElongator &&
@@ -194,7 +197,7 @@ export const twitterFetch = async (
       continue;
     }
 
-    const remainingRateLimit = parseInt(apiRequest.headers.get('x-rate-limit-remaining') || '0');
+    const remainingRateLimit = parseInt(apiRequest?.headers.get('x-rate-limit-remaining') || '0');
     console.log(`Remaining rate limit: ${remainingRateLimit} requests`);
     /* Running out of requests within our rate limit, let's purge the cache */
     if (!useElongator && remainingRateLimit < 10) {
@@ -247,7 +250,7 @@ export const twitterFetch = async (
 
   console.log('Twitter has repeatedly denied our requests, so we give up now');
 
-  return {};
+  return null;
 };
 
 export const fetchUser = async (
@@ -287,7 +290,7 @@ export const fetchUser = async (
       }
       return !(
         response?.data?.user?.result?.__typename !== 'User' ||
-        typeof response.data.user.result.legacy === 'undefined'
+        typeof response?.data?.user?.result?.legacy === 'undefined'
       );
       /*
       return !(

src/helpers/translate.ts

@@ -1,5 +1,6 @@
 import { Context } from 'hono';
 import { Constants } from '../constants';
+import { withTimeout } from './utils';
 
 /* Handles translating Tweets when asked! */
 export const translateTweet = async (
@@ -53,10 +54,11 @@ export const translateTweet = async (
   try {
     const url = `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`;
     console.log(url, headers);
-    translationApiResponse = await c.env?.TwitterProxy.fetch(url, {
+    translationApiResponse = await withTimeout((signal: AbortSignal) => c.env?.TwitterProxy.fetch(url, {
       method: 'GET',
-      headers: headers
-    });
+      headers: headers,
+      signal: signal
+    })) as Response;
     translationResults = (await translationApiResponse.json()) as TranslationPartial;
 
     console.log(`translationResults`, translationResults);
@@ -69,6 +71,6 @@ export const translateTweet = async (
     return translationResults;
   } catch (e: unknown) {
     console.error('Unknown error while fetching from Translation API', e);
-    return {} as TranslationPartial; // No work to do
+    return null;
   }
 };

src/helpers/utils.ts

@@ -32,6 +32,29 @@ export const truncateWithEllipsis = (str: string, maxLength: number): string =>
   return truncated.length < str.length ? truncated + '…' : truncated;
 };
 
+export async function withTimeout<T>(
+  asyncTask: (signal: AbortSignal) => Promise<T>, 
+  timeout: number = 3000
+): Promise<T> {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  try {
+      const result = await asyncTask(controller.signal);
+      /* Clear the timeout if the task completes in time */
+      clearTimeout(timeoutId); 
+      console.log(`Clearing timeout after ${timeout}ms`);
+      return result;
+  } catch (error) {
+      if ((error as Error).name === 'AbortError') {
+          throw new Error('Asynchronous task was aborted due to timeout');
+      } else {
+        /* Re-throw other errors for further handling */
+          throw error as Error; 
+      }
+  }
+}
+
 const numberFormat = new Intl.NumberFormat('en-US');
 
 export const formatNumber = (num: number) => numberFormat.format(num);

src/providers/twitter/conversation.ts

@@ -62,15 +62,14 @@ export const fetchTweetDetail = async (
       const conversation = _conversation as TweetDetailResult;
       const tweet = findTweetInBucket(
         status,
-        processResponse(conversation.data?.threaded_conversation_with_injections_v2?.instructions)
+        processResponse(conversation?.data?.threaded_conversation_with_injections_v2?.instructions)
       );
       if (tweet && isGraphQLTweet(tweet)) {
         return true;
       }
       console.log('invalid graphql tweet', conversation);
       const firstInstruction = (
-        conversation.data?.threaded_conversation_with_injections_v2
-          .instructions?.[0] as TimelineAddEntriesInstruction
+        conversation?.data?.threaded_conversation_with_injections_v2?.instructions?.[0] as TimelineAddEntriesInstruction
       )?.entries?.[0];
       if (
         (
@@ -297,8 +296,7 @@ export const constructTwitterThread = async (
     console.log(response);
 
     const firstInstruction = (
-      response.data?.threaded_conversation_with_injections_v2
-        .instructions?.[0] as TimelineAddEntriesInstruction
+      response?.data?.threaded_conversation_with_injections_v2?.instructions?.[0] as TimelineAddEntriesInstruction
     )?.entries?.[0];
     if (
       (
@@ -338,7 +336,7 @@ export const constructTwitterThread = async (
   }
 
   const bucket = processResponse(
-    response.data.threaded_conversation_with_injections_v2.instructions
+    response?.data?.threaded_conversation_with_injections_v2?.instructions ?? []
   );
   const originalTweet = findTweetInBucket(id, bucket);
 
@@ -482,7 +480,7 @@ export const constructTwitterThread = async (
         break;
       }
       const cursorResponse = processResponse(
-        loadCursor.data.threaded_conversation_with_injections_v2.instructions
+        loadCursor?.data?.threaded_conversation_with_injections_v2.instructions
       );
       bucket.tweets = cursorResponse.tweets.concat(
         filterBucketTweets(bucket.tweets, originalTweet)

src/providers/twitter/profile.ts

@@ -70,9 +70,13 @@ export const convertToApiUser = (user: GraphQLUser, legacyAPI = false): APIUser
 const populateUserProperties = async (
   response: GraphQLUserResponse,
   legacyAPI = false
-): Promise<APIUser> => {
-  const user = response.data.user.result;
+): Promise<APIUser | null> => {
+  const user = response?.data?.user?.result;
+  if (user) {
     return convertToApiUser(user, legacyAPI);
+  }
+  
+  return null;
 };
 
 /* API for Twitter profiles (Users)
@@ -95,7 +99,7 @@ export const userAPI = async (
   const apiUser: APIUser = (await populateUserProperties(userResponse, true)) as APIUser;
 
   /* Currently, we haven't rolled this out as it's part of the proto-v2 API */
-  delete apiUser.global_screen_name;
+  delete apiUser?.global_screen_name;
 
   /* Finally, staple the User to the response and return it */
   response.user = apiUser;

src/strings.ts

@@ -31,15 +31,15 @@ export const Strings = {
 ███   Worker build ${RELEASE_NAME}
 
 --><head>{headers}</head><body>{body}</body></html>`,
-  ERROR_HTML: `<!DOCTYPE html>
-  <html lang="en">
+ERROR_HTML: `<!DOCTYPE html>
+<html lang="en">
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <meta content="${BRANDING_NAME}" property="og:title"/>
     <meta content="Owie, you crashed ${BRANDING_NAME} :(
 
-This is caused by Twitter API downtime or a new bug. Try again in a little while." property="og:description"/></head>
+This may be caused by API downtime or a new bug. Try again in a little while." property="og:description"/></head>
     <title>:(</title>
     <style>
       body {
@@ -62,7 +62,39 @@ This is caused by Twitter API downtime or a new bug. Try again in a little while
     <h2>You hit a snag that broke ${BRANDING_NAME}. It's not your fault though&mdash;This is usually caused by a Twitter outage or a new bug.</h2>
     <p>${RELEASE_NAME}</p>
   </body>
-  </html>`
+</html>`
+  .replace(/( {2})/g, '')
+  .replace(/>\s+</gm, '><'),
+  TIMEOUT_ERROR_HTML: `<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta content="${BRANDING_NAME}" property="og:title"/>
+    <meta content="A downstream timeout occurred while trying to generate the embed. Please try again in a little while." property="og:description"/></head>
+    <title>:(</title>
+    <style>
+      body {
+        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+        padding: 0 20px;
+      }
+      h1 {
+        font-size: 4em;
+        font-weight: 900;
+        margin-bottom: 0;
+      }
+      p {
+        font-size: 10px;
+        opacity: 0.3;
+      }
+    </style>
+  </head>
+  <body>
+    <h1>Gateway Timeout</h1>
+    <h2>A downstream timeout occurred while trying to generate the embed. Please try again in a little while.</h2>
+    <p>${RELEASE_NAME}</p>
+  </body>
+</html>`
     .replace(/( {2})/g, '')
     .replace(/>\s+</gm, '><'),
   VERSION_HTML: `<!DOCTYPE html>

src/worker.ts

@@ -8,6 +8,7 @@ import { Constants } from './constants';
 import { api } from './realms/api/router';
 import { twitter } from './realms/twitter/router';
 import { cacheMiddleware } from './caches';
+import { withTimeout } from './helpers/utils';
 
 const noCache = 'max-age=0, no-cache, no-store, must-revalidate';
 
@@ -79,7 +80,15 @@ app.use('*', async (c, next) => {
 app.onError((err, c) => {
   c.get('sentry')?.captureException?.(err);
   console.error(err.stack);
-  c.status(200);
+  let errorCode = 500;
+  if (err.name === 'AbortError') {
+    errorCode = 504;
+  }
+  /* We return it as a 200 so embedded applications can display the error */
+  if (c.req.header('User-Agent')?.match(/(discordbot|telegrambot|facebook|whatsapp|firefox\/92|vkshare)/gi)) {
+    errorCode = 200;
+  }
+  c.status(errorCode);
   c.header('cache-control', noCache);
 
   return c.html(Strings.ERROR_HTML);
@@ -107,25 +116,41 @@ app.route(`/twitter`, twitter);
 
 app.all('/error', async c => {
   c.header('cache-control', noCache);
+  
+  if (c.req.header('User-Agent')?.match(/(discordbot|telegrambot|facebook|whatsapp|firefox\/92|vkshare)/gi)) {
+    c.status(200);
+    return c.html(Strings.ERROR_HTML);
+  }
   c.status(400);
+  /* We return it as a 200 so embedded applications can display the error */
   return c.body('');
 });
 
 export default {
   async fetch(request: Request, env: Env, ctx: ExecutionContext) {
     try {
-      return await app.fetch(request, env, ctx);
+      return await withTimeout(async () => app.fetch(request, env, ctx), 10);
     } catch (err) {
       console.error(err);
+      const e = err as Error;
       console.log(`Ouch, that error hurt so much Sentry couldn't catch it`);
+      console.log(e.stack);
+      let errorCode = 500;
+      if (e.name === 'AbortError') {
+        errorCode = 504;
+      }
+      /* We return it as a 200 so embedded applications can display the error */
+      if (request.headers.get('user-agent')?.match(/(discordbot|telegrambot|facebook|whatsapp|firefox\/92|vkshare)/gi)) {
+        errorCode = 200;
+      }
 
-      return new Response(Strings.ERROR_HTML, {
+      return new Response(e.name === 'AbortError' ? Strings.TIMEOUT_ERROR_HTML : Strings.ERROR_HTML, {
         headers: {
           ...Constants.RESPONSE_HEADERS,
           'content-type': 'text/html;charset=utf-8',
           'cache-control': noCache
         },
-        status: 200
+        status: errorCode
       });
     }
   }