From fc5bc49a85d97653519b63a920b63074949eaf3b Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Sun, 9 Apr 2023 18:04:36 -0400 Subject: [PATCH] Replace api fallback domain with elongator --- .env.example | 3 +-- jestconfig.json | 1 - src/api/status.ts | 14 ++++++-------- src/constants.ts | 1 - src/fetch.ts | 35 ++++++++++++++++++++++++----------- src/types/env.d.ts | 3 ++- webpack.config.js | 3 +-- wrangler.example.toml | 3 +++ 8 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.env.example b/.env.example index 11eb342..fb2d5f8 100644 --- a/.env.example +++ b/.env.example @@ -12,5 +12,4 @@ EMBED_URL = "https://github.com/FixTweet/FixTweet" SENTRY_DSN = "" SENTRY_AUTH_TOKEN = "" SENTRY_ORG = "" -SENTRY_PROJECT = "" -API_FALLBACK_DOMAIN = "https://" \ No newline at end of file +SENTRY_PROJECT = "" \ No newline at end of file diff --git a/jestconfig.json b/jestconfig.json index c33ba92..863ec8b 100644 --- a/jestconfig.json +++ b/jestconfig.json @@ -16,7 +16,6 @@ "REDIRECT_URL": "https://github.com/FixTweet/FixTweet", "EMBED_URL": "https://github.com/FixTweet/FixTweet", "RELEASE_NAME": "fixtweet-test", - "API_FALLBACK_DOMAIN": "", "TEST": true }, "testRegex": "/test/.*\\.test\\.ts$", diff --git a/src/api/status.ts b/src/api/status.ts index 5821b63..56b8188 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -155,25 +155,23 @@ export const statusAPI = async ( tweet = conversation?.globalObjects?.tweets?.[tweet.retweeted_status_id_str] || {}; } - /* Fallback for if Tweet did not load */ + /* Fallback for if Tweet did not load (i.e. NSFW) */ if (typeof tweet.full_text === 'undefined') { console.log('Invalid status, got tweet ', tweet, ' conversation ', conversation); - /* We've got timeline instructions, so the Tweet is probably private */ if (conversation.timeline?.instructions?.length > 0) { - // Try request again with fallback this time - console.log('No Tweet was found, trying fallback in case of geo-restriction'); + /* Try again using elongator API proxy */ + console.log('No Tweet was found, loading again from elongator'); conversation = await fetchConversation(status, event, true); tweet = conversation?.globalObjects?.tweets?.[status] || {}; if (typeof tweet.full_text !== 'undefined') { - console.log('Successfully loaded Tweet, requiring fallback'); - } - - if ( + console.log('Successfully loaded Tweet using elongator'); + } else if ( typeof tweet.full_text === 'undefined' && conversation.timeline?.instructions?.length > 0 ) { + console.log('Tweet could not be accessed with elongator, must be private/suspended'); return { code: 401, message: 'PRIVATE_TWEET' }; } } else { diff --git a/src/constants.ts b/src/constants.ts index b21815d..689571b 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,7 +15,6 @@ export const Constants = { API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`, TWITTER_ROOT: 'https://twitter.com', TWITTER_API_ROOT: 'https://api.twitter.com', - API_FALLBACK_DOMAIN: API_FALLBACK_DOMAIN, BOT_UA_REGEX: /bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node/gi, /* 3 hours */ diff --git a/src/fetch.ts b/src/fetch.ts index da2cc79..0b20c9e 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -6,6 +6,7 @@ const API_ATTEMPTS = 16; export const twitterFetch = async ( url: string, event: FetchEvent, + useElongator = false, validateFunction: (response: unknown) => boolean ): Promise => { let apiAttempts = 0; @@ -84,14 +85,18 @@ export const twitterFetch = async ( This can effectively mean virtually unlimited (read) access to Twitter's API, which is very funny. */ - activate = await fetch(guestTokenRequest.clone()); + // if (useElongator) { + // activate = await TwitterProxy.fetch(guestTokenRequest.clone()); + // } else { + activate = await fetch(guestTokenRequest.clone()); + // } } /* Let's grab that guest_token so we can use it */ let activateJson: { guest_token: string }; try { - activateJson = (await activate.clone().json()) as { guest_token: string }; + activateJson = (await activate?.clone().json()) as { guest_token: string }; } catch (e: unknown) { continue; } @@ -117,11 +122,20 @@ export const twitterFetch = async ( let apiRequest; try { - apiRequest = await fetch(url, { - method: 'GET', - headers: headers - }); - response = await apiRequest.json(); + if (useElongator && typeof TwitterProxy !== 'undefined') { + console.log('Fetching using elongator'); + apiRequest = await TwitterProxy.fetch(url, { + method: 'GET', + headers: headers + }); + } else { + apiRequest = await fetch(url, { + method: 'GET', + headers: headers + }); + } + + response = await apiRequest?.json(); } catch (e: unknown) { /* We'll usually only hit this if we get an invalid response from Twitter. It's uncommon, but it happens */ @@ -177,13 +191,12 @@ export const twitterFetch = async ( export const fetchConversation = async ( status: string, event: FetchEvent, - fallback = false + useElongator = false ): Promise => { return (await twitterFetch( - `${ - fallback ? Constants.API_FALLBACK_DOMAIN : Constants.TWITTER_API_ROOT - }/2/timeline/conversation/${status}.json?${Constants.GUEST_FETCH_PARAMETERS}`, + `${Constants.TWITTER_API_ROOT}/2/timeline/conversation/${status}.json?${Constants.GUEST_FETCH_PARAMETERS}`, event, + useElongator, (_conversation: unknown) => { const conversation = _conversation as TimelineBlobPartial; return !( diff --git a/src/types/env.d.ts b/src/types/env.d.ts index 0d008bf..c131a8c 100644 --- a/src/types/env.d.ts +++ b/src/types/env.d.ts @@ -9,9 +9,10 @@ declare const EMBED_URL: string; declare const REDIRECT_URL: string; declare const MOSAIC_DOMAIN_LIST: string; declare const API_HOST_LIST: string; -declare const API_FALLBACK_DOMAIN: string; declare const SENTRY_DSN: string; declare const RELEASE_NAME: string; declare const TEST: boolean | undefined; + +declare const TwitterProxy: { fetch: (input: RequestInfo>, init?: RequestInit | undefined) => Promise }; \ No newline at end of file diff --git a/webpack.config.js b/webpack.config.js index 40b7321..b3a9c1c 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -29,8 +29,7 @@ let envVariables = [ 'API_HOST_LIST', 'SENTRY_DSN', 'DEPRECATED_DOMAIN_LIST', - 'DEPRECATED_DOMAIN_EPOCH', - 'API_FALLBACK_DOMAIN' + 'DEPRECATED_DOMAIN_EPOCH' ]; let plugins = [ diff --git a/wrangler.example.toml b/wrangler.example.toml index d4bd0a5..f048818 100644 --- a/wrangler.example.toml +++ b/wrangler.example.toml @@ -3,6 +3,9 @@ account_id = "[CLOUDFLARE_ACCOUNT_ID]" main = "./dist/worker.js" compatibility_date = "2022-08-17" send_metrics = false +services = [ + { binding = "TwitterProxy", service = "elongator" } +] [build] command = "npm run build"