From 4ae10034db605a080d1041a5debfa2ab1ab39914 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 18 Aug 2022 00:49:57 -0400 Subject: [PATCH 1/5] switch on guest token cache (borked) --- src/api.ts | 5 +++-- src/fetch.ts | 5 ++--- src/server.ts | 3 ++- src/status.ts | 5 +++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/api.ts b/src/api.ts index 90bb46e..5ae648e 100644 --- a/src/api.ts +++ b/src/api.ts @@ -138,9 +138,10 @@ const populateTweetProperties = async ( available for free using api.fxtwitter.com. */ export const statusAPI = async ( status: string, - language: string | undefined + language: string | undefined, + event: FetchEvent ): Promise => { - const conversation = await fetchUsingGuest(status); + const conversation = await fetchUsingGuest(status, event); const tweet = conversation?.globalObjects?.tweets?.[status] || {}; /* Fallback for if Tweet did not load */ diff --git a/src/fetch.ts b/src/fetch.ts index ef9f6c8..09786e9 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,6 +1,6 @@ import { Constants } from './constants'; -export const fetchUsingGuest = async (status: string): Promise => { +export const fetchUsingGuest = async (status: string, event: FetchEvent): Promise => { let apiAttempts = 0; let cachedTokenFailed = false; @@ -121,9 +121,8 @@ export const fetchUsingGuest = async (status: string): Promise => { console.log('Direct?', flags?.direct); - const api = await statusAPI(status, language); + const api = await statusAPI(status, language, event as FetchEvent); const tweet = api?.tweet as APITweet; /* Catch this request if it's an API response */ From 7707c0d2d06437352a8b27d99f546e246f11f56e Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 18 Aug 2022 03:09:41 -0400 Subject: [PATCH 2/5] Implement Cloudflare guest token caching --- src/fetch.ts | 61 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index 09786e9..7abb8ca 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,8 +1,11 @@ import { Constants } from './constants'; -export const fetchUsingGuest = async (status: string, event: FetchEvent): Promise => { +export const fetchUsingGuest = async ( + status: string, + event: FetchEvent +): Promise => { let apiAttempts = 0; - let cachedTokenFailed = false; + let newTokenGenerated = false; const tokenHeaders: { [header: string]: string } = { Authorization: Constants.GUEST_BEARER_TOKEN, @@ -14,11 +17,21 @@ export const fetchUsingGuest = async (status: string, event: FetchEvent): Promis { method: 'POST', headers: tokenHeaders, + body: '' + } + ); + + /* A dummy version of the request only used for Cloudflare caching purposes. + The reason it exists at all is because Cloudflare won't cache POST requests. */ + const guestTokenRequestCacheDummy = new Request( + `${Constants.TWITTER_API_ROOT}/1.1/guest/activate.json`, + { + method: 'GET', + headers: tokenHeaders, cf: { cacheEverything: true, - cacheTtl: 600 - }, - body: '' + cacheTtl: 300 + } } ); @@ -41,19 +54,19 @@ export const fetchUsingGuest = async (status: string, event: FetchEvent): Promis let activate: Response | null = null; - if (!cachedTokenFailed) { - const cachedResponse = await cache.match(guestTokenRequest); + if (!newTokenGenerated) { + const cachedResponse = await cache.match(guestTokenRequestCacheDummy); if (cachedResponse) { console.log('Token cache hit'); activate = cachedResponse; + } else { + console.log('Token cache miss'); + newTokenGenerated = true; } - - console.log('Token cache miss'); - cachedTokenFailed = true; } - if (cachedTokenFailed || activate === null) { + if (newTokenGenerated || activate === null) { /* If all goes according to plan, we have a guest token we can use to call API AFAIK there is no limit to how many guest tokens you can request. @@ -66,14 +79,14 @@ export const fetchUsingGuest = async (status: string, event: FetchEvent): Promis let activateJson: { guest_token: string }; try { - activateJson = (await activate.json()) as { guest_token: string }; + activateJson = (await activate.clone().json()) as { guest_token: string }; } catch (e: unknown) { continue; } const guestToken = activateJson.guest_token; - console.log('Activated guest:', activateJson); + console.log(newTokenGenerated ? 'Activated guest:' : 'Using guest:', activateJson); console.log('Guest token:', guestToken); /* Just some cookies to mimick what the Twitter Web App would send */ @@ -107,10 +120,21 @@ export const fetchUsingGuest = async (status: string, event: FetchEvent): Promis /* We'll usually only hit this if we get an invalid response from Twitter. It's uncommon, but it happens */ console.error('Unknown error while fetching conversation from API'); - cachedTokenFailed = true; + event && event.waitUntil(cache.delete(guestTokenRequestCacheDummy)); + newTokenGenerated = true; continue; } + const remainingRateLimit = parseInt( + apiRequest.headers.get('x-rate-limit-remaining') || '0' + ); + console.log(`Remaining rate limit: ${remainingRateLimit} requests`); + /* Running out of requests within our rate limit, let's purge the cache */ + if (remainingRateLimit < 20) { + console.log(`Purging token on this edge due to low rate limit remaining`); + event && event.waitUntil(cache.delete(guestTokenRequestCacheDummy)); + } + if ( typeof conversation.globalObjects === 'undefined' && (typeof conversation.errors === 'undefined' || @@ -118,11 +142,14 @@ export const fetchUsingGuest = async (status: string, event: FetchEvent): Promis 239) /* TODO: i forgot what code 239 actually is lol */ ) { console.log('Failed to fetch conversation, got', conversation); - cachedTokenFailed = true; + newTokenGenerated = true; continue; } - /* Once we've confirmed we have a working guest token, let's cache it! */ - event.waitUntil(cache.put(guestTokenRequest, activate.clone())); + console.log('Caching guest token'); + /* If we've generated a new token, we'll cache it */ + if (newTokenGenerated) { + event && event.waitUntil(cache.put(guestTokenRequestCacheDummy, activate.clone())); + } conversation.guestToken = guestToken; return conversation; } From f49846d4c5344024436da67bdb840c9ab80f901a Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 18 Aug 2022 03:58:25 -0400 Subject: [PATCH 3/5] Some tweaks to maybe make it work better --- src/fetch.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/fetch.ts b/src/fetch.ts index 7abb8ca..73d14bf 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -17,6 +17,10 @@ export const fetchUsingGuest = async ( { method: 'POST', headers: tokenHeaders, + cf: { + cacheEverything: true, + cacheTtl: 300 + }, body: '' } ); @@ -27,7 +31,6 @@ export const fetchUsingGuest = async ( `${Constants.TWITTER_API_ROOT}/1.1/guest/activate.json`, { method: 'GET', - headers: tokenHeaders, cf: { cacheEverything: true, cacheTtl: 300 @@ -145,10 +148,10 @@ export const fetchUsingGuest = async ( newTokenGenerated = true; continue; } - console.log('Caching guest token'); /* If we've generated a new token, we'll cache it */ - if (newTokenGenerated) { - event && event.waitUntil(cache.put(guestTokenRequestCacheDummy, activate.clone())); + if (event && newTokenGenerated) { + console.log('Caching guest token'); + event.waitUntil(cache.put(guestTokenRequestCacheDummy, activate.clone())); } conversation.guestToken = guestToken; return conversation; From c2a617d804c951746ffc557b6eef7d9c7b7655a5 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 18 Aug 2022 04:08:15 -0400 Subject: [PATCH 4/5] Fix issues running miniflare directly --- wrangler.example.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wrangler.example.toml b/wrangler.example.toml index 7cc6e7c..d4bd0a5 100644 --- a/wrangler.example.toml +++ b/wrangler.example.toml @@ -5,4 +5,7 @@ compatibility_date = "2022-08-17" send_metrics = false [build] -command = "npm run build" \ No newline at end of file +command = "npm run build" + +[miniflare.globals] +TEST = "true" # Will have unicode character errors in headers if not set to true \ No newline at end of file From 01b3b8d79b65750f0571c9725de1bcf391d28554 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 18 Aug 2022 11:11:16 -0400 Subject: [PATCH 5/5] Fix borked guest token caching in workers runtime --- src/fetch.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/fetch.ts b/src/fetch.ts index 73d14bf..ecd8385 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -150,8 +150,14 @@ export const fetchUsingGuest = async ( } /* If we've generated a new token, we'll cache it */ if (event && newTokenGenerated) { + const cachingResponse = new Response(await activate.clone().text(), { + headers: { + ...tokenHeaders, + 'cache-control': 'max-age=300' + } + }); console.log('Caching guest token'); - event.waitUntil(cache.put(guestTokenRequestCacheDummy, activate.clone())); + event.waitUntil(cache.put(guestTokenRequestCacheDummy, cachingResponse)); } conversation.guestToken = guestToken; return conversation;