DevComp/fxtwitter-docker
1
0
Fork 0
mirror of https://github.com/CompeyDev/fxtwitter-docker.git synced 2025-04-05 10:30:55 +01:00
Code Issues Projects Releases Packages Wiki Activity

Convert worker to Hono and ES Module

Browse source
This commit is contained in:
dangered wolf 2023-11-09 03:53:24 -05:00
parent 396954a4ae
commit 01da30b9e8
No known key found for this signature in database
GPG key ID: 41E4D37680ED8B58
24 changed files with 746 additions and 797 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
esbuild.config.mjs
View file

@ -60,7 +60,7 @@ await esbuild.build({
entryPoints: ['src/worker.ts'],
sourcemap: 'external',
outdir: 'dist',
minify: true,
minify: false,
bundle: true,
format: 'esm',
plugins: [

21
package-lock.json generated
View file

@ -9,9 +9,8 @@
"version": "1.0.0",
"license": "MIT",
"dependencies": {
"hono": "^3.9.2",
"itty-router": "^4.0.23",
"toucan-js": "^3.3.1"
"@hono/sentry": "^1.0.0",
"hono": "^3.9.2"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20231025.0",
@ -1241,6 +1240,17 @@
"node": ">=14"
}
},
"node_modules/@hono/sentry": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@hono/sentry/-/sentry-1.0.0.tgz",
"integrity": "sha512-GbPxgpGuasM2zRCSaA77MPWu4KDcuk/EMf7JJykjCvnOTbjmtr7FovNxsvg7xlXCIjZDgLmqBaoJMi3AxbeIAA==",
"dependencies": {
"toucan-js": "^3.2.2"
},
"peerDependencies": {
"hono": "3.*"
}
},
"node_modules/@humanwhocodes/config-array": {
"version": "0.11.13",
"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.13.tgz",
@ -4549,11 +4559,6 @@
"node": ">=8"
}
},
"node_modules/itty-router": {
"version": "4.0.23",
"resolved": "https://registry.npmjs.org/itty-router/-/itty-router-4.0.23.tgz",
"integrity": "sha512-tP1NI8PVK43vWlBnIPqj47ni5FDSczFviA4wgBznscndo8lEvBA+pO3DD1rNbIQPcZhprr775iUTunyGvQMcBw=="
},
"node_modules/jest": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz",

5
package.json
View file

@ -38,8 +38,7 @@
"wrangler": "^3.15.0"
},
"dependencies": {
"hono": "^3.9.2",
"itty-router": "^4.0.23",
"toucan-js": "^3.3.1"
"@hono/sentry": "^1.0.0",
"hono": "^3.9.2"
}
}

79
src/caches.ts Normal file
View file

@ -0,0 +1,79 @@
import { MiddlewareHandler } from 'hono';
import { Constants } from './constants';
import {} from 'hono';
/* Wrapper to handle caching, and misc things like catching robots.txt */
export const cacheMiddleware = (): MiddlewareHandler => async (c, next) => {
const request = c.req;
const userAgent = request.header('User-Agent') ?? '';
// https://developers.cloudflare.com/workers/examples/cache-api/
const cacheUrl = new URL(
userAgent.includes('Telegram')
? `${request.url}&telegram`
: userAgent.includes('Discord')
? `${request.url}&discord`
: request.url
);
console.log('cacheUrl', cacheUrl);
const cacheKey = new Request(cacheUrl.toString(), request);
const cache = caches.default;
switch (request.method) {
case 'GET':
if (
!Constants.API_HOST_LIST.includes(cacheUrl.hostname) &&
!request.header('Cookie')?.includes('base_redirect')
) {
/* cache may be undefined in tests */
const cachedResponse = await cache.match(cacheKey);
if (cachedResponse) {
console.log('Cache hit');
return new Response(cachedResponse.body, cachedResponse);
}
console.log('Cache miss');
}
await next();
// eslint-disable-next-line no-case-declarations
const response = c.res.clone();
/* Store the fetched response as cacheKey
Use waitUntil so you can return the response without blocking on
writing to cache */
try {
c.executionCtx.waitUntil(cache.put(cacheKey, response.clone()));
} catch (error) {
console.error((error as Error).stack);
}
return response;
/* Telegram sends this from Webpage Bot, and Cloudflare sends it if we purge cache, and we respect it.
PURGE is not defined in an RFC, but other servers like Nginx apparently use it.
Update 2023-11-09:
For some reason, even before migrating to Hono, this returns 403 Forbidden now when PURGEd.
I'm not sure why, as this is clearly not what we are doing. Is Cloudflare doing this? Is something else wrong? We'll also accept DELETE to do the same I guess. */
case 'PURGE':
case 'DELETE':
console.log('Purging cache as requested');
await cache.delete(cacheKey);
return c.text('')
/* yes, we do give HEAD */
case 'HEAD':
return c.text('')
/* We properly state our OPTIONS when asked */
case 'OPTIONS':
c.header('allow', Constants.RESPONSE_HEADERS.allow)
c.status(204)
return c.text('');
default:
c.status(405);
return c.text('')
}
};

6
src/constants.ts
View file

@ -59,8 +59,10 @@ export const Constants = {
RESPONSE_HEADERS: {
'allow': 'OPTIONS, GET, PURGE, HEAD',
'content-type': 'text/html;charset=UTF-8',
'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`,
'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets
'x-powered-by': `${RELEASE_NAME}`,
'x-trans-rights': 'true',
'cache-control': 'max-age=3600', // Can be overriden in some cases, like unfinished poll tweets
'Vary': 'Accept-Encoding, User-Agent'
},
API_RESPONSE_HEADERS: {
'access-control-allow-origin': '*',

71
src/embed/status.ts
View file

@ -7,45 +7,41 @@ import { renderPhoto } from '../render/photo';
import { renderVideo } from '../render/video';
import { renderInstantView } from '../render/instantview';
import { constructTwitterThread } from '../providers/twitter/conversation';
import { IRequest } from 'itty-router';
import { Context } from 'hono';
export const returnError = (error: string): StatusResponse => {
return {
text: Strings.BASE_HTML.format({
lang: '',
headers: [
`<meta property="og:title" content="${Constants.BRANDING_NAME}"/>`,
`<meta property="og:description" content="${error}"/>`
].join('')
})
};
export const returnError = (c: Context, error: string): Response => {
return c.text(Strings.BASE_HTML.format({
lang: '',
headers: [
`<meta property="og:title" content="${Constants.BRANDING_NAME}"/>`,
`<meta property="og:description" content="${error}"/>`
].join('')
}));
};
/* Handler for Twitter statuses (Tweets).
Like Twitter, we use the terminologies interchangably. */
export const handleStatus = async (
c: Context,
status: string,
mediaNumber: number | undefined,
userAgent: string,
flags: InputFlags,
language: string,
event: FetchEvent,
request: IRequest
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<StatusResponse> => {
): Promise<Response> => {
console.log('Direct?', flags?.direct);
let fetchWithThreads = false;
/* TODO: Enable actually pulling threads once we can actually do something with them */
if (request?.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) {
if (c.req.header('user-agent')?.includes('Telegram') && !flags?.direct) {
fetchWithThreads = false;
}
const thread = await constructTwitterThread(
status,
fetchWithThreads,
request,
c,
language,
flags?.api ?? false
);
@ -76,27 +72,27 @@ export const handleStatus = async (
/* Catch this request if it's an API response */
if (flags?.api) {
return {
response: new Response(JSON.stringify(api), {
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS },
status: api.code
})
};
c.status(api.code);
// Add every header from Constants.API_RESPONSE_HEADERS
for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) {
c.header(header, value);
}
return c.text(JSON.stringify(api));
}
if (tweet === null) {
return returnError(Strings.ERROR_TWEET_NOT_FOUND);
return returnError(c, Strings.ERROR_TWEET_NOT_FOUND);
}
/* If there was any errors fetching the Tweet, we'll return it */
switch (api.code) {
case 401:
return returnError(Strings.ERROR_PRIVATE);
return returnError(c, Strings.ERROR_PRIVATE);
case 404:
return returnError(Strings.ERROR_TWEET_NOT_FOUND);
return returnError(c, Strings.ERROR_TWEET_NOT_FOUND);
case 500:
console.log(api);
return returnError(Strings.ERROR_API_FAIL);
return returnError(c, Strings.ERROR_API_FAIL);
}
const isTelegram = (userAgent || '').indexOf('Telegram') > -1;
@ -146,7 +142,7 @@ export const handleStatus = async (
}
if (redirectUrl) {
return { response: Response.redirect(redirectUrl, 302) };
return c.redirect(redirectUrl, 302);
}
}
@ -157,7 +153,6 @@ export const handleStatus = async (
const engagementText = authorText.replace(/ {4}/g, ' ');
let siteName = Constants.BRANDING_NAME;
let newText = tweet.text;
let cacheControl: string | null = null;
/* Base headers included in all responses */
const headers = [
@ -217,7 +212,8 @@ export const handleStatus = async (
console.log('overrideMedia', JSON.stringify(overrideMedia));
if (!flags?.textOnly) {
const media = tweet.media?.all && tweet.media?.all.length > 0 ? tweet.media : tweet.quote?.media || {}
const media =
tweet.media?.all && tweet.media?.all.length > 0 ? tweet.media : tweet.quote?.media || {};
if (overrideMedia) {
let instructions: ResponseInstructions;
@ -340,7 +336,7 @@ export const handleStatus = async (
Yes, checking if this is a string is a hacky way to do this, but
it can do it in way less code than actually comparing dates */
if (poll.time_left_en !== 'Final results') {
cacheControl = Constants.POLL_TWEET_CACHE;
c.header('cache-control', Constants.POLL_TWEET_CACHE);
}
/* And now we'll put the poll right after the Tweet text! */
@ -427,12 +423,9 @@ export const handleStatus = async (
const lang = tweet.lang === null ? 'en' : tweet.lang || 'en';
/* Finally, after all that work we return the response HTML! */
return {
text: Strings.BASE_HTML.format({
lang: `lang="${lang}"`,
headers: headers.join(''),
body: ivbody
}).replace(/>(\s+)</gm, '><'), // Remove whitespace between tags
cacheControl: cacheControl
};
return c.text(Strings.BASE_HTML.format({
lang: `lang="${lang}"`,
headers: headers.join(''),
body: ivbody
}).replace(/>(\s+)</gm, '><'));
};

30
src/fetch.ts
View file

@ -1,3 +1,4 @@
import { Context } from 'hono';
import { Constants } from './constants';
import { Experiment, experimentCheck } from './experiments';
import { generateUserAgent } from './helpers/useragent';
@ -19,11 +20,11 @@ const generateSnowflake = () => {
globalThis.fetchCompletedTime = 0;
export const twitterFetch = async (
c: Context,
url: string,
event: FetchEvent,
useElongator = experimentCheck(
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
typeof c.env.TwitterProxy !== 'undefined'
),
validateFunction: (response: unknown) => boolean,
elongatorRequired = false
@ -140,10 +141,10 @@ export const twitterFetch = async (
let apiRequest;
try {
if (useElongator && typeof TwitterProxy !== 'undefined') {
if (useElongator && typeof c.env.TwitterProxy !== 'undefined') {
console.log('Fetching using elongator');
const performanceStart = performance.now();
apiRequest = await TwitterProxy.fetch(url, {
apiRequest = await c.env.TwitterProxy.fetch(url, {
method: 'GET',
headers: headers
});
@ -170,8 +171,8 @@ export const twitterFetch = async (
return {};
}
!useElongator &&
event &&
event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }));
c.executionCtx &&
c.executionCtx.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }));
if (useElongator) {
console.log('Elongator request failed, trying again without it');
wasElongatorDisabled = true;
@ -186,7 +187,7 @@ export const twitterFetch = async (
if (
!wasElongatorDisabled &&
!useElongator &&
typeof TwitterProxy !== 'undefined' &&
typeof c.env.TwitterProxy !== 'undefined' &&
(response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason ===
'NsfwLoggedOut'
) {
@ -200,8 +201,8 @@ export const twitterFetch = async (
/* Running out of requests within our rate limit, let's purge the cache */
if (!useElongator && remainingRateLimit < 10) {
console.log(`Purging token on this edge due to low rate limit remaining`);
event &&
event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }));
c.executionCtx &&
c.executionCtx.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }));
}
if (!validateFunction(response)) {
@ -219,7 +220,7 @@ export const twitterFetch = async (
continue;
}
/* If we've generated a new token, we'll cache it */
if (event && newTokenGenerated && activate) {
if (c.executionCtx && newTokenGenerated && activate) {
const cachingResponse = new Response(await activate.clone().text(), {
headers: {
...tokenHeaders,
@ -227,7 +228,7 @@ export const twitterFetch = async (
}
});
console.log('Caching guest token');
event.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse));
c.executionCtx.waitUntil(cache.put(guestTokenRequestCacheDummy.clone(), cachingResponse));
}
// @ts-expect-error - We'll pin the guest token to whatever response we have
@ -243,13 +244,13 @@ export const twitterFetch = async (
export const fetchUser = async (
username: string,
event: FetchEvent,
c: Context,
useElongator = experimentCheck(
Experiment.ELONGATOR_PROFILE_API,
typeof TwitterProxy !== 'undefined'
typeof c.env.TwitterProxy !== 'undefined'
)
): Promise<GraphQLUserResponse> => {
return (await twitterFetch(
return (await twitterFetch(c,
`${
Constants.TWITTER_ROOT
}/i/api/graphql/sLVLhk0bGj3MVFEKTdax1w/UserByScreenName?variables=${encodeURIComponent(
@ -266,7 +267,6 @@ export const fetchUser = async (
verified_phone_label_enabled: true
})
)}`,
event,
useElongator,
// Validator function
(_res: unknown) => {

4
src/helpers/graphql.ts
View file

@ -17,6 +17,8 @@ export const isGraphQLTweet = (response: unknown): response is GraphQLTweet => {
typeof response === 'object' &&
response !== null &&
// @ts-expect-error it's 6 am please let me sleep
('__typename' in response && (response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults') || ('legacy' in response && response.legacy?.full_text))
(('__typename' in response &&
(response.__typename === 'Tweet' || response.__typename === 'TweetWithVisibilityResults')) ||
('legacy' in response && response.legacy?.full_text))
);
};

8
src/helpers/translate.ts
View file

@ -1,10 +1,12 @@
import { Context } from 'hono';
import { Constants } from '../constants';
/* Handles translating Tweets when asked! */
export const translateTweet = async (
tweet: GraphQLTweet,
guestToken: string,
language: string
language: string,
c: Context
): Promise<TranslationPartial | null> => {
const csrfToken = crypto.randomUUID().replace(/-/g, ''); // Generate a random CSRF token, this doesn't matter, Twitter just cares that header and cookie match
@ -44,14 +46,14 @@ export const translateTweet = async (
headers['x-twitter-client-language'] = language;
/* As of August 2023, you can no longer fetch translations with guest token */
if (typeof TwitterProxy === 'undefined') {
if (typeof c.env.TwitterProxy === 'undefined') {
return null;
}
try {
const url = `${Constants.TWITTER_ROOT}/i/api/1.1/strato/column/None/tweetId=${tweet.rest_id},destinationLanguage=None,translationSource=Some(Google),feature=None,timeout=None,onlyCached=None/translation/service/translateTweet`;
console.log(url, headers);
translationApiResponse = await TwitterProxy.fetch(url, {
translationApiResponse = await c.env.TwitterProxy.fetch(url, {
method: 'GET',
headers: headers
});

1
src/helpers/utils.ts
View file

@ -32,7 +32,6 @@ export const truncateWithEllipsis = (str: string, maxLength: number): string =>
return truncated.length < str.length ? truncated + '…' : truncated;
};
const numberFormat = new Intl.NumberFormat('en-US');
export const formatNumber = (num: number) => numberFormat.format(num);

53
src/providers/twitter/conversation.ts
View file

@ -1,17 +1,17 @@
import { IRequest } from 'itty-router';
import { Constants } from '../../constants';
import { twitterFetch } from '../../fetch';
import { buildAPITweet } from './processor';
import { Experiment, experimentCheck } from '../../experiments';
import { isGraphQLTweet } from '../../helpers/graphql';
import { Context } from 'hono';
export const fetchTweetDetail = async (
c: Context,
status: string,
event: FetchEvent,
useElongator = typeof TwitterProxy !== 'undefined',
useElongator = typeof c.env.TwitterProxy !== 'undefined',
cursor: string | null = null
): Promise<TweetDetailResult> => {
return (await twitterFetch(
return (await twitterFetch(c,
`${
Constants.TWITTER_ROOT
}/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent(
@ -56,7 +56,6 @@ export const fetchTweetDetail = async (
withArticleRichContentState: true
})
)}`,
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as TweetDetailResult;
@ -90,13 +89,13 @@ export const fetchTweetDetail = async (
export const fetchByRestId = async (
status: string,
event: FetchEvent,
c: Context,
useElongator = experimentCheck(
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
typeof c.env.TwitterProxy !== 'undefined'
)
): Promise<TweetResultsByRestIdResult> => {
return (await twitterFetch(
return (await twitterFetch(c,
`${
Constants.TWITTER_ROOT
}/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent(
@ -133,7 +132,6 @@ export const fetchByRestId = async (
withArticleRichContentState: true
})
)}`,
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as TweetResultsByRestIdResult;
@ -273,7 +271,7 @@ const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
export const constructTwitterThread = async (
id: string,
processThread = false,
request: IRequest,
c: Context,
language: string | undefined,
legacyAPI = false
): Promise<SocialThread> => {
@ -286,9 +284,13 @@ export const constructTwitterThread = async (
Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why.
I'll figure out why eventually, but for now just don't use TweetDetail for this. */
if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) {
if (
typeof c.env.TwitterProxy !== 'undefined' &&
!language &&
(experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)
) {
console.log('Using TweetDetail for request...');
response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult;
response = (await fetchTweetDetail(c, id)) as TweetDetailResult;
console.log(response);
@ -312,7 +314,7 @@ export const constructTwitterThread = async (
/* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */
if (!response) {
console.log('Using TweetResultsByRestId for request...');
response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult;
response = (await fetchByRestId(id, c)) as TweetResultsByRestIdResult;
const result = response?.data?.tweetResult?.result as GraphQLTweet;
@ -320,7 +322,7 @@ export const constructTwitterThread = async (
return { post: null, thread: null, author: null, code: 404 };
}
const buildPost = await buildAPITweet(result, language, false, legacyAPI);
const buildPost = await buildAPITweet(c, result, language, false, legacyAPI);
if ((buildPost as FetchResults)?.status === 401) {
return { post: null, thread: null, author: null, code: 401 };
@ -343,7 +345,7 @@ export const constructTwitterThread = async (
return { post: null, thread: null, author: null, code: 404 };
}
post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet;
post = (await buildAPITweet(c, originalTweet, undefined, false, legacyAPI)) as APITweet;
if (post === null) {
return { post: null, thread: null, author: null, code: 404 };
@ -400,7 +402,7 @@ export const constructTwitterThread = async (
let loadCursor: TweetDetailResult;
try {
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value);
loadCursor = await fetchTweetDetail(c, id, true, cursor.value);
if (
typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions ===
@ -464,7 +466,7 @@ export const constructTwitterThread = async (
let loadCursor: TweetDetailResult;
try {
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value);
loadCursor = await fetchTweetDetail(c, id, true, cursor.value);
if (
typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions ===
@ -501,18 +503,21 @@ export const constructTwitterThread = async (
};
threadTweets.forEach(async tweet => {
socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet);
socialThread.thread?.push((await buildAPITweet(c, tweet, undefined, true, false)) as APITweet);
});
return socialThread;
};
export const threadAPIProvider = async (request: IRequest) => {
const { id } = request.params;
export const threadAPIProvider = async (c: Context) => {
const id = c.req.param('id') as string;
const processedResponse = await constructTwitterThread(id, true, request, undefined);
const processedResponse = await constructTwitterThread(id, true, c, undefined);
return new Response(JSON.stringify(processedResponse), {
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }
});
c.status(processedResponse.code);
// Add every header from Constants.API_RESPONSE_HEADERS
for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) {
c.header(header, value);
}
return c.text(JSON.stringify(processedResponse));
};

18
src/providers/twitter/processor.ts
View file

@ -7,12 +7,14 @@ import { unescapeText } from '../../helpers/utils';
import { processMedia } from '../../helpers/media';
import { convertToApiUser } from './profile';
import { translateTweet } from '../../helpers/translate';
import { Context } from 'hono';
export const buildAPITweet = async (
c: Context,
tweet: GraphQLTweet,
language: string | undefined,
threadPiece = false,
legacyAPI = false
legacyAPI = false,
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<APITweet | FetchResults | null> => {
const apiTweet = {} as APITweet;
@ -149,19 +151,13 @@ export const buildAPITweet = async (
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
const buildQuote = (await buildAPITweet(
quoteTweet,
language,
threadPiece,
legacyAPI
));
const buildQuote = await buildAPITweet(c, quoteTweet, language, threadPiece, legacyAPI);
if ((buildQuote as FetchResults).status) {
apiTweet.quote = undefined
apiTweet.quote = undefined;
} else {
apiTweet.quote = buildQuote as APITweet;
}
/* Only override the embed_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.embed_card === 'tweet' && typeof apiTweet.quote !== 'undefined') {
apiTweet.embed_card = apiTweet.quote.embed_card;
@ -237,12 +233,12 @@ export const buildAPITweet = async (
apiTweet.embed_card = 'player';
}
console.log('language?', language)
console.log('language?', language);
/* If a language is specified in API or by user, let's try translating it! */
if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
console.log(`Attempting to translate Tweet to ${language}...`);
const translateAPI = await translateTweet(tweet, '', language);
const translateAPI = await translateTweet(tweet, '', language, c);
if (translateAPI !== null && translateAPI?.translation) {
apiTweet.translation = {
text: unescapeText(

5
src/providers/twitter/profile.ts
View file

@ -1,3 +1,4 @@
import { Context } from 'hono';
import { Constants } from '../../constants';
import { fetchUser } from '../../fetch';
import { linkFixer } from '../../helpers/linkFixer';
@ -79,10 +80,10 @@ const populateUserProperties = async (
available for free using api.fxtwitter.com. */
export const userAPI = async (
username: string,
event: FetchEvent
c: Context
// flags?: InputFlags
): Promise<UserAPIResponse> => {
const userResponse = await fetchUser(username, event);
const userResponse = await fetchUser(username, c);
if (!userResponse || !Object.keys(userResponse).length) {
return {
code: 404,

24
src/realms/api/router.ts Normal file
View file

@ -0,0 +1,24 @@
import { Hono } from 'hono';
import { statusRequest } from '../twitter/routes/status';
import { profileRequest } from '../twitter/routes/profile';
import { Strings } from '../../strings';
export const api = new Hono();
/* Current v1 API endpoints. Currently, these still go through the Twitter embed requests. API v2+ won't do this. */
api.get('/:handle?/status/:id/:language?', statusRequest);
api.get(
'/:handle?/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?',
statusRequest
);
api.get('/:handle', profileRequest);
api.get(
'/robots.txt',
async (c) => {
c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate');
c.status(200);
return c.text(Strings.ROBOTS_TXT);
}
);

29
src/realms/common/version.ts Normal file
View file

@ -0,0 +1,29 @@
import { Context } from 'hono';
import { Constants } from '../../constants';
import { sanitizeText } from '../../helpers/utils';
import { Strings } from '../../strings';
export const versionRoute = async (context: Context) => {
const request = context.req;
return new Response(
Strings.VERSION_HTML.format({
rtt: request.raw.cf?.clientTcpRtt ? `🏓 ${request.raw.cf.clientTcpRtt} ms RTT` : '',
colo: (request.raw.cf?.colo as string) ?? '??',
httpversion: (request.raw.cf?.httpProtocol as string) ?? 'Unknown HTTP Version',
tlsversion: (request.raw.cf?.tlsVersion as string) ?? 'Unknown TLS Version',
ip: request.header('x-real-ip') ?? request.header('cf-connecting-ip') ?? 'Unknown IP',
city: (request.raw.cf?.city as string) ?? 'Unknown City',
region: (request.raw.cf?.region as string) ?? request.raw.cf?.country ?? 'Unknown Region',
country: (request.raw.cf?.country as string) ?? 'Unknown Country',
asn: `AS${request.raw.cf?.asn ?? '??'} (${request.raw.cf?.asOrganization ?? 'Unknown ASN'})`,
ua: sanitizeText(request.header('user-agent') ?? 'Unknown User Agent')
}),
{
headers: {
...Constants.RESPONSE_HEADERS,
'cache-control': 'max-age=0, no-cache, no-store, must-revalidate'
},
status: 200
}
);
};

58
src/realms/twitter/router.ts Normal file
View file

@ -0,0 +1,58 @@
import { Context, Hono } from 'hono';
// import { cache } from "hono/cache";
import { versionRoute } from '../common/version';
import { Strings } from '../../strings';
import { Constants } from '../../constants';
import { genericTwitterRedirect, setRedirectRequest } from './routes/redirects';
import { profileRequest } from './routes/profile';
import { statusRequest } from './routes/status';
import { oembed } from './routes/oembed';
export const twitter = new Hono();
twitter.get('/status/:id', statusRequest);
// twitter.get('/:handle/status/:id', statusRequest);
// twitter.get('/:prefix/:handle/status/:id/:language?', statusRequest);
// twitter.get(
// '/:prefix/:handle/status/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?',
// statusRequest
// );
// twitter.get('/:handle?/:endpoint{status(es)?}/:id/:language?', statusRequest);
// twitter.get(
// '/:handle?/:endpoint{status(es)?}/:id/:mediaType{(photos?|videos?)}/:mediaNumber{[1-4]}/:language?',
// statusRequest
// );
twitter.get('/version', versionRoute);
twitter.get('/set_base_redirect', setRedirectRequest);
twitter.get('/oembed', oembed);
twitter.get(
'/robots.txt',
async (c) => {
c.header('cache-control', 'max-age=0, no-cache, no-store, must-revalidate');
c.status(200);
return c.text(Strings.ROBOTS_TXT);
}
);
twitter.get('/i/events/:id', genericTwitterRedirect);
twitter.get('/hashtag/:hashtag', genericTwitterRedirect);
twitter.get('/:handle', profileRequest);
export const getBaseRedirectUrl = (c: Context) => {
const baseRedirect = c.req.header('cookie')?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0];
if (baseRedirect) {
console.log('Found base redirect', baseRedirect);
try {
new URL(baseRedirect);
} catch (e) {
return Constants.TWITTER_ROOT;
}
return baseRedirect.endsWith('/') ? baseRedirect.slice(0, -1) : baseRedirect;
}
return Constants.TWITTER_ROOT;
};

34
src/realms/twitter/routes/oembed.ts Normal file
View file

@ -0,0 +1,34 @@
import { Context } from 'hono';
import motd from '../../../../motd.json';
import { Constants } from '../../../constants';
import { Strings } from '../../../strings';
/* Yes, I actually made the endpoint /owoembed. Deal with it. */
export const oembed = async (c: Context) => {
console.log('oembed hit!');
const { searchParams } = new URL(c.req.url);
/* Fallbacks */
const text = searchParams.get('text') || 'Twitter';
const author = searchParams.get('author') || 'jack';
const status = searchParams.get('status') || '20';
const random = Math.floor(Math.random() * Object.keys(motd).length);
const [name, url] = Object.entries(motd)[random];
const test = {
author_name: text,
author_url: `${Constants.TWITTER_ROOT}/${encodeURIComponent(author)}/status/${status}`,
/* Change provider name if tweet is on deprecated domain. */
provider_name:
searchParams.get('deprecated') === 'true' ? Strings.DEPRECATED_DOMAIN_NOTICE_DISCORD : name,
provider_url: url,
title: Strings.DEFAULT_AUTHOR_TEXT,
type: 'link',
version: '1.0'
};
c.header('content-type', 'application/json')
c.status(200);
/* Stringify and send it on its way! */
return c.text(JSON.stringify(test))
};

70
src/realms/twitter/routes/profile.ts Normal file
View file

@ -0,0 +1,70 @@
import { Context } from 'hono';
import { Constants } from '../../../constants';
import { handleProfile } from '../../../user';
import { getBaseRedirectUrl } from '../router';
/* Handler for User Profiles */
export const profileRequest = async (c: Context) => {
const handle = c.req.param('handle');
const url = new URL(c.req.url);
const userAgent = c.req.header('User-Agent') || '';
const flags: InputFlags = {};
/* User Agent matching for embed generators, bots, crawlers, and other automated
tools. It's pretty all-encompassing. Note that Firefox/92 is in here because
Discord sometimes uses the following UA:
Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0
I'm not sure why that specific one, it's pretty weird, but this edge case ensures
stuff keeps working.
On the very rare off chance someone happens to be using specifically Firefox 92,
the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */
const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null;
/* If not a valid screen name, we redirect to project GitHub */
if (handle.match(/\w{1,15}/gi)?.[0] !== handle) {
return c.redirect(Constants.REDIRECT_URL, 302);
}
const username = handle.match(/\w{1,15}/gi)?.[0] as string;
/* Check if request is to api.fxtwitter.com */
if (Constants.API_HOST_LIST.includes(url.hostname)) {
console.log('JSON API request');
flags.api = true;
}
const baseUrl = getBaseRedirectUrl(c);
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
if (cacheControl) {
c.header('cache-control', cacheControl);
}
/* Direct media or API access bypasses bot check, returning same response regardless of UA */
if (isBotUA || flags.api) {
if (isBotUA) {
console.log(`Matched bot UA ${userAgent}`);
} else {
console.log('Bypass bot check');
}
/* This throws the necessary data to handleStatus (in status.ts) */
const profileResponse = await handleProfile(c, username, flags);
/* Check for custom redirect */
if (!isBotUA) {
return c.redirect(`${baseUrl}/${handle}`, 302);
}
/* Return the response containing embed information */
return profileResponse;
} else {
/* A human has clicked a fxtwitter.com/:screen_name link!
Obviously we just need to redirect to the user directly.*/
console.log('Matched human UA', userAgent);
return c.redirect(`${baseUrl}/${handle}`, 302)
}
};

74
src/realms/twitter/routes/redirects.ts Normal file
View file

@ -0,0 +1,74 @@
import { Context } from 'hono';
import { Strings } from '../../../strings';
import { sanitizeText } from '../../../helpers/utils';
import { getBaseRedirectUrl } from '../router';
import { Constants } from '../../../constants';
export const genericTwitterRedirect = async (c: Context) => {
const url = new URL(c.req.url);
const baseUrl = getBaseRedirectUrl(c);
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
if (cacheControl) {
c.header('cache-control', cacheControl);
}
return c.redirect(`${baseUrl}${url.pathname}`, 302);
};
export const setRedirectRequest = async (c: Context) => {
/* Query params */
const { searchParams } = new URL(c.req.url);
let url = searchParams.get('url');
/* Check that origin either does not exist or is in our domain list */
const origin = c.req.header('origin');
if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) {
c.status(403);
return c.text(Strings.MESSAGE_HTML.format({
message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.`
}))
}
if (!url) {
/* Remove redirect URL */
// eslint-disable-next-line sonarjs/no-duplicate-string
c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`);
// eslint-disable-next-line sonarjs/no-duplicate-string
c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`);
c.status(200);
return c.text(Strings.MESSAGE_HTML.format({
message: `Your base redirect has been cleared. To set one, please pass along the <code>url</code> parameter.`
}))
}
try {
new URL(url);
} catch (e) {
try {
new URL(`https://${url}`);
} catch (e) {
/* URL is not well-formed, remove */
console.log('Invalid base redirect URL, removing cookie before redirect');
c.header('set-cookie', `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`);
c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`);
c.status(200);
return c.text(Strings.MESSAGE_HTML.format({
message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net`
}))
}
url = `https://${url}`;
}
c.header('set-cookie', `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`);
c.header('content-security-policy', `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`);
return c.text(Strings.MESSAGE_HTML.format({
message: `Successfully set base redirect, you will now be redirected to ${sanitizeText(url)} rather than ${Constants.TWITTER_ROOT}`
}))
};

151
src/realms/twitter/routes/status.ts Normal file
View file

@ -0,0 +1,151 @@
import { Context } from 'hono';
import { Constants } from '../../../constants';
import { getBaseRedirectUrl } from '../router';
import { handleStatus } from '../../../embed/status';
import { Strings } from '../../../strings';
/* Handler for status (Tweet) request */
export const statusRequest = async (c: Context) => {
const handle = c.req.param('handle');
const id = c.req.param('id');
const mediaNumber = c.req.param('mediaNumber');
const language = c.req.param('language');
const prefix = c.req.param('prefix');
const url = new URL(c.req.url);
const flags: InputFlags = {};
// eslint-disable-next-line sonarjs/no-duplicate-string
const userAgent = c.req.header('User-Agent') || '';
/* Let's return our HTML version for wayback machine (we can add other archivers too in future) */
if (
['archive.org', 'Wayback Machine'].some(service => c.req.header('Via')?.includes?.(service))
) {
console.log('Request from archive.org');
flags.archive = true;
}
/* User Agent matching for embed generators, bots, crawlers, and other automated
tools. It's pretty all-encompassing. Note that Firefox/92 is in here because
Discord sometimes uses the following UA:
Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0
I'm not sure why that specific one, it's pretty weird, but this edge case ensures
stuff keeps working.
On the very rare off chance someone happens to be using specifically Firefox 92,
the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */
const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null || flags?.archive;
/* Check if domain is a direct media domain (i.e. d.fxtwitter.com),
the tweet is prefixed with /dl/ or /dir/ (for TwitFix interop), or the
tweet ends in .mp4, .jpg, .jpeg, or .png
Note that .png is not documented because images always redirect to a jpg,
but it will help someone who does it mistakenly on something like Discord
Also note that all we're doing here is setting the direct flag. If someone
links a video and ends it with .jpg, it will still redirect to a .mp4! */
if (url.pathname.match(/\/status(es)?\/\d{2,20}\.(mp4|png|jpe?g)/g)) {
console.log('Direct media request by extension');
flags.direct = true;
} else if (Constants.DIRECT_MEDIA_DOMAINS.includes(url.hostname)) {
console.log('Direct media request by domain');
flags.direct = true;
} else if (Constants.TEXT_ONLY_DOMAINS.includes(url.hostname)) {
console.log('Text-only embed request');
flags.textOnly = true;
} else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) {
console.log('Forced instant view request');
flags.forceInstantView = true;
} else if (prefix === 'dl' || prefix === 'dir') {
console.log('Direct media request by path prefix');
flags.direct = true;
}
/* The pxtwitter.com domain is deprecated and Tweets posted after deprecation
date will have a notice saying we've moved to fxtwitter.com! */
if (
Constants.DEPRECATED_DOMAIN_LIST.includes(url.hostname) &&
BigInt(id?.match(/\d{2,20}/g)?.[0] || 0) > Constants.DEPRECATED_DOMAIN_EPOCH
) {
console.log('Request to deprecated domain');
flags.deprecated = true;
}
/* TODO: Figure out what we're doing with FixTweet / FixupX branding in future */
if (/fixup/g.test(url.href)) {
console.log(`We're using x domain`);
flags.isXDomain = true;
} else {
console.log(`We're using twitter domain`);
}
const baseUrl = getBaseRedirectUrl(c);
/* Check if request is to api.fxtwitter.com, or the tweet is appended with .json
Note that unlike TwitFix, FixTweet will never generate embeds for .json, and
in fact we only support .json because it's what people using TwitFix API would
be used to. */
if (
url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null ||
Constants.API_HOST_LIST.includes(url.hostname)
) {
console.log('JSON API request');
flags.api = true;
}
/* Direct media or API access bypasses bot check, returning same response regardless of UA */
if (isBotUA || flags.direct || flags.api) {
if (isBotUA) {
console.log(`Matched bot UA ${userAgent}`);
} else {
console.log('Bypass bot check');
}
/* This throws the necessary data to handleStatus (in status.ts) */
const statusResponse = await handleStatus(c,
id?.match(/\d{2,20}/)?.[0] || '0',
mediaNumber ? parseInt(mediaNumber) : undefined,
userAgent,
flags,
language
);
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
if (cacheControl) {
// eslint-disable-next-line sonarjs/no-duplicate-string
c.header('cache-control', cacheControl);
}
if (statusResponse) {
/* We're checking if the User Agent is a bot again specifically in case they requested
direct media (d.fxtwitter.com, .mp4/.jpg, etc) but the Tweet contains no media.
Since we obviously have no media to give the user, we'll just redirect to the Tweet.
Embeds will return as usual to bots as if direct media was never specified. */
if (!isBotUA && !flags.api) {
const baseUrl = getBaseRedirectUrl(c);
return c.redirect(`${baseUrl}/${handle || 'i'}/status/${id}`, 302);
}
c.status(200);
/* Return the response containing embed information */
return statusResponse;
} else {
/* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */
c.status(500);
return c.text(Strings.ERROR_UNKNOWN);
}
} else {
/* A human has clicked a fxtwitter.com/:screen_name/status/:id link!
Obviously we just need to redirect to the Tweet directly.*/
console.log('Matched human UA', userAgent);
c.redirect(`${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`, 302);
return c.text('');
}
};

14
src/strings.ts
View file

@ -203,20 +203,6 @@ This is caused by Twitter API downtime or a new bug. Try again in a little while
# Check out the docs at https://${API_HOST_LIST.split(',')[0]} to learn how to use it
# ________________
# / /|
# / / |
# /_______________/ |
# | ___________ | / |
# | | | | / |
# | | | | / |
# | | gaming | | / |
# | |__________| | / |
# | | / |
# | _____ | / |
# | _____________ | /
# |_____________| /
# Good luck, have fun and try not to take over the world!
# Instructions below are for robots only, beep boop

6
src/types/types.d.ts vendored
View file

@ -37,12 +37,6 @@ interface RenderProperties {
flags?: InputFlags;
}
interface Request {
params: {
[param: string]: string;
};
}
interface TweetAPIResponse {
code: number;
message: string;

57
src/user.ts
View file

@ -1,50 +1,48 @@
import { Context } from 'hono';
import { Constants } from './constants';
import { Strings } from './strings';
import { userAPI } from './providers/twitter/profile';
export const returnError = (error: string): StatusResponse => {
return {
text: Strings.BASE_HTML.format({
lang: '',
headers: [
`<meta property="og:title" content="${Constants.BRANDING_NAME}"/>`,
`<meta property="og:description" content="${error}"/>`
].join('')
})
};
export const returnError = (c: Context, error: string): Response => {
return c.text(Strings.BASE_HTML.format({
lang: '',
headers: [
`<meta property="og:title" content="${Constants.BRANDING_NAME}"/>`,
`<meta property="og:description" content="${error}"/>`
].join('')
}));
};
/* Handler for Twitter users */
export const handleProfile = async (
c: Context,
username: string,
userAgent?: string,
flags?: InputFlags,
event?: FetchEvent
): Promise<StatusResponse> => {
flags: InputFlags,
): Promise<Response> => {
console.log('Direct?', flags?.direct);
const api = await userAPI(username, event as FetchEvent);
const api = await userAPI(username, c);
const user = api?.user as APIUser;
/* Catch this request if it's an API response */
// For now we just always return the API response while testing
if (flags?.api) {
return {
response: new Response(JSON.stringify(api), {
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS },
status: api.code
})
};
c.status(api.code);
// Add every header from Constants.API_RESPONSE_HEADERS
for (const [header, value] of Object.entries(Constants.API_RESPONSE_HEADERS)) {
c.header(header, value);
}
return c.text(JSON.stringify(api));
}
/* If there was any errors fetching the User, we'll return it */
switch (api.code) {
case 401:
return returnError(Strings.ERROR_PRIVATE);
return returnError(c, Strings.ERROR_PRIVATE);
case 404:
return returnError(Strings.ERROR_USER_NOT_FOUND);
return returnError(c, Strings.ERROR_USER_NOT_FOUND);
case 500:
return returnError(Strings.ERROR_API_FAIL);
return returnError(c, Strings.ERROR_API_FAIL);
}
/* Base headers included in all responses */
@ -53,11 +51,8 @@ export const handleProfile = async (
// TODO Add card creation logic here
/* Finally, after all that work we return the response HTML! */
return {
text: Strings.BASE_HTML.format({
lang: `lang="en"`,
headers: headers.join('')
}),
cacheControl: null
};
return c.text(Strings.BASE_HTML.format({
lang: `lang="en"`,
headers: headers.join('')
}));
};

721
src/worker.ts
View file

@ -1,663 +1,114 @@
/* eslint-disable no-case-declarations */
import { Toucan } from 'toucan-js';
import { Env, Hono } from 'hono';
import { timing } from 'hono/timing';
import { logger } from 'hono/logger';
import { RewriteFrames } from '@sentry/integrations';
import { IRequest, Router } from 'itty-router';
import { Constants } from './constants';
import { handleStatus } from './embed/status';
import { sentry } from '@hono/sentry';
import { Strings } from './strings';
import { Constants } from './constants';
import { api } from './realms/api/router';
import { twitter } from './realms/twitter/router';
import { cacheMiddleware } from './caches';
import motd from '../motd.json';
import { sanitizeText } from './helpers/utils';
import { handleProfile } from './user';
// import { threadAPIProvider } from './providers/twitter/conversation';
const noCache = 'max-age=0, no-cache, no-store, must-revalidate';
declare const globalThis: {
fetchCompletedTime: number;
};
const app = new Hono<{ Bindings: { TwitterProxy: Fetcher; AnalyticsEngine: AnalyticsEngineDataset } }>({
getPath: req => {
let url: URL;
const router = Router();
const getBaseRedirectUrl = (request: IRequest) => {
const baseRedirect = request.headers
?.get('cookie')
?.match(/(?<=base_redirect=)(.*?)(?=;|$)/)?.[0];
if (baseRedirect) {
console.log('Found base redirect', baseRedirect);
try {
new URL(baseRedirect);
} catch (e) {
return Constants.TWITTER_ROOT;
url = new URL(req.url);
} catch (err) {
return '/error';
}
return baseRedirect.endsWith('/') ? baseRedirect.slice(0, -1) : baseRedirect;
}
return Constants.TWITTER_ROOT;
};
/* Handler for status (Tweet) request */
const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputFlags = {}) => {
const { handle, id, mediaNumber, language, prefix } = request.params;
const url = new URL(request.url);
// eslint-disable-next-line sonarjs/no-duplicate-string
const userAgent = request.headers.get('User-Agent') || '';
/* Let's return our HTML version for wayback machine (we can add other archivers too in future) */
if (
['archive.org', 'Wayback Machine'].some(
service => request.headers.get('Via')?.includes?.(service)
)
) {
console.log('Request from archive.org');
flags.archive = true;
}
/* User Agent matching for embed generators, bots, crawlers, and other automated
tools. It's pretty all-encompassing. Note that Firefox/92 is in here because
Discord sometimes uses the following UA:
Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0
I'm not sure why that specific one, it's pretty weird, but this edge case ensures
stuff keeps working.
On the very rare off chance someone happens to be using specifically Firefox 92,
the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */
const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null || flags?.archive;
/* Check if domain is a direct media domain (i.e. d.fxtwitter.com),
the tweet is prefixed with /dl/ or /dir/ (for TwitFix interop), or the
tweet ends in .mp4, .jpg, .jpeg, or .png
Note that .png is not documented because images always redirect to a jpg,
but it will help someone who does it mistakenly on something like Discord
Also note that all we're doing here is setting the direct flag. If someone
links a video and ends it with .jpg, it will still redirect to a .mp4! */
if (url.pathname.match(/\/status(es)?\/\d{2,20}\.(mp4|png|jpe?g)/g)) {
console.log('Direct media request by extension');
flags.direct = true;
} else if (Constants.DIRECT_MEDIA_DOMAINS.includes(url.hostname)) {
console.log('Direct media request by domain');
flags.direct = true;
} else if (Constants.TEXT_ONLY_DOMAINS.includes(url.hostname)) {
console.log('Text-only embed request');
flags.textOnly = true;
} else if (Constants.INSTANT_VIEW_DOMAINS.includes(url.hostname)) {
console.log('Forced instant view request');
flags.forceInstantView = true;
} else if (prefix === 'dl' || prefix === 'dir') {
console.log('Direct media request by path prefix');
flags.direct = true;
}
/* The pxtwitter.com domain is deprecated and Tweets posted after deprecation
date will have a notice saying we've moved to fxtwitter.com! */
if (
Constants.DEPRECATED_DOMAIN_LIST.includes(url.hostname) &&
BigInt(id?.match(/\d{2,20}/g)?.[0] || 0) > Constants.DEPRECATED_DOMAIN_EPOCH
) {
console.log('Request to deprecated domain');
flags.deprecated = true;
}
/* TODO: Figure out what we're doing with FixTweet / FixupX branding in future */
if (/fixup/g.test(url.href)) {
console.log(`We're using x domain`);
flags.isXDomain = true;
} else {
console.log(`We're using twitter domain`);
}
const baseUrl = getBaseRedirectUrl(request);
/* Check if request is to api.fxtwitter.com, or the tweet is appended with .json
Note that unlike TwitFix, FixTweet will never generate embeds for .json, and
in fact we only support .json because it's what people using TwitFix API would
be used to. */
if (
url.pathname.match(/\/status(es)?\/\d{2,20}\.(json)/g) !== null ||
Constants.API_HOST_LIST.includes(url.hostname)
) {
console.log('JSON API request');
flags.api = true;
}
/* Direct media or API access bypasses bot check, returning same response regardless of UA */
if (isBotUA || flags.direct || flags.api) {
if (isBotUA) {
console.log(`Matched bot UA ${userAgent}`);
} else {
console.log('Bypass bot check');
const baseHostName = url.hostname.split('.').slice(-2).join('.');
let realm = 'twitter';
/* Override if in API_HOST_LIST. Note that we have to check full hostname for this. */
if (Constants.API_HOST_LIST.includes(url.hostname)) {
realm = 'api';
} else if (Constants.STANDARD_DOMAIN_LIST.includes(baseHostName)) {
realm = 'twitter';
}
/* Defaults to Twitter realm if unknown domain specified (such as the *.workers.dev hostname or deprecated domain) */
/* This throws the necessary data to handleStatus (in status.ts) */
const statusResponse = await handleStatus(
id?.match(/\d{2,20}/)?.[0] || '0',
mediaNumber ? parseInt(mediaNumber) : undefined,
userAgent,
flags,
language,
event,
request
);
/* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */
if (statusResponse.response) {
console.log('handleStatus sent response');
return statusResponse.response;
} else if (statusResponse.text) {
console.log('handleStatus sent embed');
/* We're checking if the User Agent is a bot again specifically in case they requested
direct media (d.fxtwitter.com, .mp4/.jpg, etc) but the Tweet contains no media.
Since we obviously have no media to give the user, we'll just redirect to the Tweet.
Embeds will return as usual to bots as if direct media was never specified. */
if (!isBotUA && !flags.api) {
const baseUrl = getBaseRedirectUrl(request);
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
return new Response(null, {
status: 302,
headers: {
Location: `${baseUrl}/${handle || 'i'}/status/${id}`,
...(cacheControl ? { 'cache-control': cacheControl } : {})
}
});
}
let headers = Constants.RESPONSE_HEADERS;
if (statusResponse.cacheControl) {
headers = {
...headers,
'cache-control':
baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : statusResponse.cacheControl
};
}
/* Return the response containing embed information */
return new Response(statusResponse.text, {
headers: headers,
status: 200
});
} else {
/* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */
return new Response(Strings.ERROR_UNKNOWN, {
headers: Constants.RESPONSE_HEADERS,
status: 500
});
}
} else {
globalThis.fetchCompletedTime = performance.now();
/* A human has clicked a fxtwitter.com/:screen_name/status/:id link!
Obviously we just need to redirect to the Tweet directly.*/
console.log('Matched human UA', userAgent);
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
return new Response(null, {
status: 302,
headers: {
Location: `${baseUrl}/${handle || 'i'}/status/${id?.match(/\d{2,20}/)?.[0]}`,
...(cacheControl ? { 'cache-control': cacheControl } : {})
}
});
console.log(`/${realm}${url.pathname}`);
return `/${realm}${url.pathname}`;
}
};
});
/* Handler for User Profiles */
const profileRequest = async (request: IRequest, event: FetchEvent, flags: InputFlags = {}) => {
const { handle } = request.params;
const url = new URL(request.url);
const userAgent = request.headers.get('User-Agent') || '';
/* User Agent matching for embed generators, bots, crawlers, and other automated
tools. It's pretty all-encompassing. Note that Firefox/92 is in here because
Discord sometimes uses the following UA:
Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6; rv:92.0) Gecko/20100101 Firefox/92.0
I'm not sure why that specific one, it's pretty weird, but this edge case ensures
stuff keeps working.
On the very rare off chance someone happens to be using specifically Firefox 92,
the http-equiv="refresh" meta tag will ensure an actual human is sent to the destination. */
const isBotUA = userAgent.match(Constants.BOT_UA_REGEX) !== null;
/* If not a valid screen name, we redirect to project GitHub */
if (handle.match(/\w{1,15}/gi)?.[0] !== handle) {
return Response.redirect(Constants.REDIRECT_URL, 302);
}
const username = handle.match(/\w{1,15}/gi)?.[0] as string;
/* Check if request is to api.fxtwitter.com */
if (Constants.API_HOST_LIST.includes(url.hostname)) {
console.log('JSON API request');
flags.api = true;
}
/* Direct media or API access bypasses bot check, returning same response regardless of UA */
if (isBotUA || flags.api) {
if (isBotUA) {
console.log(`Matched bot UA ${userAgent}`);
} else {
console.log('Bypass bot check');
}
/* This throws the necessary data to handleStatus (in status.ts) */
const profileResponse = await handleProfile(username, userAgent, flags, event);
/* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */
if (profileResponse.response) {
console.log('handleProfile sent response');
return profileResponse.response;
} else if (profileResponse.text) {
console.log('handleProfile sent embed');
/* TODO This check has purpose in the original handleStatus handler, but I'm not sure if this edge case can happen here */
const baseUrl = getBaseRedirectUrl(request);
/* Check for custom redirect */
if (!isBotUA) {
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
return new Response(null, {
status: 302,
headers: {
Location: `${baseUrl}/${handle}`,
...(cacheControl ? { 'cache-control': cacheControl } : {})
}
});
}
let headers = Constants.RESPONSE_HEADERS;
if (profileResponse.cacheControl) {
headers = {
...headers,
'cache-control':
baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : profileResponse.cacheControl
};
}
/* Return the response containing embed information */
return new Response(profileResponse.text, {
headers: headers,
status: 200
});
} else {
/* Somehow handleStatus sent us nothing. This should *never* happen, but we have a case for it. */
return new Response(Strings.ERROR_UNKNOWN, {
headers: { ...Constants.RESPONSE_HEADERS, 'cache-control': 'max-age=0' },
status: 500
});
}
} else {
/* A human has clicked a fxtwitter.com/:screen_name link!
Obviously we just need to redirect to the user directly.*/
console.log('Matched human UA', userAgent);
const baseUrl = getBaseRedirectUrl(request);
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
return new Response(null, {
status: 302,
headers: {
Location: `${baseUrl}/${handle}`,
...(cacheControl ? { 'cache-control': cacheControl } : {})
}
});
}
};
const genericTwitterRedirect = async (request: IRequest) => {
const url = new URL(request.url);
const baseUrl = getBaseRedirectUrl(request);
/* Do not cache if using a custom redirect */
const cacheControl = baseUrl !== Constants.TWITTER_ROOT ? 'max-age=0' : undefined;
return new Response(null, {
status: 302,
headers: {
Location: `${baseUrl}${url.pathname}`,
...(cacheControl ? { 'cache-control': cacheControl } : {})
}
});
};
const versionRequest = async (request: IRequest) => {
globalThis.fetchCompletedTime = performance.now();
return new Response(
Strings.VERSION_HTML.format({
rtt: request.cf?.clientTcpRtt ? `🏓 ${request.cf.clientTcpRtt} ms RTT` : '',
colo: (request.cf?.colo as string) ?? '??',
httpversion: (request.cf?.httpProtocol as string) ?? 'Unknown HTTP Version',
tlsversion: (request.cf?.tlsVersion as string) ?? 'Unknown TLS Version',
ip:
request.headers.get('x-real-ip') ?? request.headers.get('cf-connecting-ip') ?? 'Unknown IP',
city: (request.cf?.city as string) ?? 'Unknown City',
region: (request.cf?.region as string) ?? request.cf?.country ?? 'Unknown Region',
country: (request.cf?.country as string) ?? 'Unknown Country',
asn: `AS${request.cf?.asn ?? '??'} (${request.cf?.asOrganization ?? 'Unknown ASN'})`,
ua: sanitizeText(request.headers.get('user-agent') ?? 'Unknown User Agent')
}),
{
headers: {
...Constants.RESPONSE_HEADERS,
'cache-control': 'max-age=0, no-cache, no-store, must-revalidate'
},
status: 200
}
);
};
const setRedirectRequest = async (request: IRequest) => {
/* Query params */
const { searchParams } = new URL(request.url);
let url = searchParams.get('url');
/* Check that origin either does not exist or is in our domain list */
const origin = request.headers.get('origin');
if (origin && !Constants.STANDARD_DOMAIN_LIST.includes(new URL(origin).hostname)) {
return new Response(
Strings.MESSAGE_HTML.format({
message: `Failed to set base redirect: Your request seems to be originating from another domain, please open this up in a new tab if you are trying to set your base redirect.`
}),
{
headers: Constants.RESPONSE_HEADERS,
status: 403
}
);
}
if (!url) {
/* Remove redirect URL */
return new Response(
Strings.MESSAGE_HTML.format({
message: `Your base redirect has been cleared. To set one, please pass along the <code>url</code> parameter.`
}),
{
headers: {
'set-cookie': `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`,
'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`,
...Constants.RESPONSE_HEADERS
},
status: 200
}
);
}
try {
new URL(url);
} catch (e) {
try {
new URL(`https://${url}`);
} catch (e) {
/* URL is not well-formed, remove */
console.log('Invalid base redirect URL, removing cookie before redirect');
return new Response(
Strings.MESSAGE_HTML.format({
message: `Your URL does not appear to be well-formed. Example: ?url=https://nitter.net`
}),
{
headers: {
'set-cookie': `base_redirect=; path=/; max-age=0; expires=Thu, 01 Jan 1970 00:00:00 GMT; secure; HttpOnly`,
'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(
' '
)};`,
...Constants.RESPONSE_HEADERS
},
status: 200
}
);
}
url = `https://${url}`;
}
/* Set cookie for url */
return new Response(
Strings.MESSAGE_HTML.format({
message: `Successfully set base redirect, you will now be redirected to ${sanitizeText(
url
)} rather than ${Constants.TWITTER_ROOT}`
}),
{
headers: {
'set-cookie': `base_redirect=${url}; path=/; max-age=63072000; secure; HttpOnly`,
'content-security-policy': `frame-ancestors ${Constants.STANDARD_DOMAIN_LIST.join(' ')};`,
...Constants.RESPONSE_HEADERS
},
status: 200
}
);
};
/* TODO: is there any way to consolidate these stupid routes for itty-router?
I couldn't find documentation allowing for regex matching */
router.get('/:prefix?/:handle/status/:id', statusRequest);
router.get('/:prefix?/:handle/status/:id/photo/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/status/:id/photos/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/status/:id/video/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/status/:id/videos/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/statuses/:id', statusRequest);
router.get('/:prefix?/:handle/statuses/:id/photo/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/statuses/:id/photos/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/statuses/:id/video/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/statuses/:id/videos/:mediaNumber', statusRequest);
router.get('/:prefix?/:handle/status/:id/:language', statusRequest);
router.get('/:prefix?/:handle/statuses/:id/:language', statusRequest);
router.get('/status/:id', statusRequest);
router.get('/status/:id/:language', statusRequest);
router.get('/version', versionRequest);
router.get('/set_base_redirect', setRedirectRequest);
// router.get('/v2/twitter/thread/:id', threadAPIProvider)
/* Oembeds (used by Discord to enhance responses)
Yes, I actually made the endpoint /owoembed. Deal with it. */
router.get('/owoembed', async (request: IRequest) => {
globalThis.fetchCompletedTime = performance.now();
console.log('oembed hit!');
const { searchParams } = new URL(request.url);
/* Fallbacks */
const text = searchParams.get('text') || 'Twitter';
const author = searchParams.get('author') || 'jack';
const status = searchParams.get('status') || '20';
const random = Math.floor(Math.random() * Object.keys(motd).length);
const [name, url] = Object.entries(motd)[random];
const test = {
author_name: text,
author_url: `${Constants.TWITTER_ROOT}/${encodeURIComponent(author)}/status/${status}`,
/* Change provider name if tweet is on deprecated domain. */
provider_name:
searchParams.get('deprecated') === 'true' ? Strings.DEPRECATED_DOMAIN_NOTICE_DISCORD : name,
provider_url: url,
title: Strings.DEFAULT_AUTHOR_TEXT,
type: 'link',
version: '1.0'
};
/* Stringify and send it on its way! */
return new Response(JSON.stringify(test), {
headers: {
...Constants.RESPONSE_HEADERS,
'content-type': 'application/json'
app.use(
'*',
sentry({
dsn: SENTRY_DSN,
requestDataOptions: {
allowedHeaders: /(.*)/,
allowedSearchParams: /(.*)/
},
status: 200
});
integrations: [new RewriteFrames({ root: '/' })],
release: RELEASE_NAME
})
);
app.use('*', async (c, next) => {
/* Apply all headers from Constants.RESPONSE_HEADERS */
for (const [header, value] of Object.entries(Constants.RESPONSE_HEADERS)) {
c.header(header, value);
}
await next();
});
/* Pass through profile requests to Twitter.
We don't currently have custom profile cards yet,
but it's something we might do. Maybe. */
router.get('/:handle', profileRequest);
router.get('/:handle/', profileRequest);
router.get('/i/events/:id', genericTwitterRedirect);
router.get('/hashtag/:hashtag', genericTwitterRedirect);
app.onError((err, c) => {
c.get('sentry').captureException(err);
/* workaround for silly TypeScript things */
const error = err as Error;
console.error(error.stack);
c.status(200);
c.header('cache-control', noCache);
c.header('content-type', 'text/html');
/* If we don't understand the route structure at all, we'll
redirect to GitHub (normal domains) or API docs (api.fxtwitter.com) */
router.get('*', async (request: IRequest) => {
const url = new URL(request.url);
if (Constants.API_HOST_LIST.includes(url.hostname)) {
return Response.redirect(Constants.API_DOCS_URL, 302);
}
return Response.redirect(Constants.REDIRECT_URL, 302);
return c.text(Strings.ERROR_HTML);
});
/* Wrapper to handle caching, and misc things like catching robots.txt */
export const cacheWrapper = async (request: Request, event?: FetchEvent): Promise<Response> => {
const startTime = performance.now();
const userAgent = request.headers.get('User-Agent') || '';
// https://developers.cloudflare.com/workers/examples/cache-api/
const cacheUrl = new URL(
userAgent.includes('Telegram')
? `${request.url}&telegram`
: userAgent.includes('Discord')
? `${request.url}&discord`
: request.url
);
app.use('*', logger());
console.log(`Hello from ⛅ ${request.cf?.colo || 'UNK'}`);
console.log('userAgent', userAgent);
console.log('cacheUrl', cacheUrl);
app.use('*', async (c, next) => {
console.log(`Hello from ⛅ ${c.req.raw.cf?.colo || 'UNK'}`);
console.log('userAgent', c.req.header('user-agent'));
await next();
});
const cacheKey = new Request(cacheUrl.toString(), request);
const cache = caches.default;
app.use('*', cacheMiddleware());
app.use('*', timing({ enabled: false }));
/* Itty-router doesn't seem to like routing file names because whatever,
so we just handle it in the caching layer instead. Kinda hacky, but whatever. */
if (cacheUrl.pathname === '/robots.txt') {
return new Response(Strings.ROBOTS_TXT, {
headers: {
...Constants.RESPONSE_HEADERS,
'content-type': 'text/plain'
},
status: 200
});
app.route(`/api`, api);
app.route(`/twitter`, twitter);
app.all(
'/error',
async (c) => {
c.header('cache-control', noCache);
c.status(400);
return c.body('')
}
);
switch (request.method) {
case 'GET':
if (
!Constants.API_HOST_LIST.includes(cacheUrl.hostname) &&
!request.headers?.get('Cookie')?.includes('base_redirect')
) {
/* cache may be undefined in tests */
const cachedResponse = await cache.match(cacheKey);
export default {
async fetch(request: Request, env: Env, ctx: ExecutionContext) {
try {
return await app.fetch(request, env, ctx);
} catch (err) {
console.error(err);
console.log('Ouch, that error hurt so much Sentry couldnt catch it');
if (cachedResponse) {
console.log('Cache hit');
return cachedResponse;
}
console.log('Cache miss');
}
const response = await router.handle(request, event);
/* Store the fetched response as cacheKey
Use waitUntil so you can return the response without blocking on
writing to cache */
try {
event && event.waitUntil(cache.put(cacheKey, response.clone()));
} catch (error) {
console.error((error as Error).stack);
}
const endTime = performance.now();
const timeSinceFetch = endTime - (globalThis.fetchCompletedTime || 0);
const timeSinceStart = endTime - startTime;
console.log(
`Request took ${timeSinceStart}ms, of which ${timeSinceFetch}ms was CPU time after last fetch`
);
return response;
/* Telegram sends this from Webpage Bot, and Cloudflare sends it if we purge cache, and we respect it.
PURGE is not defined in an RFC, but other servers like Nginx apparently use it. */
case 'PURGE':
console.log('Purging cache as requested');
await cache.delete(cacheKey);
return new Response('', { status: 200 });
/* yes, we do give HEAD */
case 'HEAD':
return new Response('', {
headers: Constants.RESPONSE_HEADERS,
return new Response(Strings.ERROR_HTML, {
headers: {
...Constants.RESPONSE_HEADERS,
'content-type': 'text/html',
'cache-control': noCache
},
status: 200
});
/* We properly state our OPTIONS when asked */
case 'OPTIONS':
return new Response('', {
headers: {
allow: Constants.RESPONSE_HEADERS.allow
},
status: 204
});
default:
return new Response('', { status: 405 });
}
}
};
/* Wrapper around Sentry, used for catching uncaught exceptions */
const sentryWrapper = async (event: FetchEvent, test = false): Promise<void> => {
let sentry: null | Toucan = null;
if (typeof SENTRY_DSN !== 'undefined' && SENTRY_DSN && !test) {
/* We use Toucan for Sentry. Toucan is a Sentry SDK designed for Cloudflare Workers / DOs */
sentry = new Toucan({
dsn: SENTRY_DSN,
context: event,
request: event.request,
requestDataOptions: {
allowedHeaders: /(.*)/,
allowedSearchParams: /(.*)/
},
/* TODO: Figure out what changed between @sentry/integration 7.65.0 and 7.66.0
https://github.com/getsentry/sentry-javascript/compare/7.65.0...7.66.0
which caused types to go apeshit */
integrations: [new RewriteFrames({ root: '/' })],
/* event includes 'waitUntil', which is essential for Sentry logs to be delivered.
Also includes 'request' -- no need to set it separately. */
release: RELEASE_NAME
});
}
/* Responds with either a returned response (good!!!) or returns
a crash response (bad!!!) */
event.respondWith(
(async (): Promise<Response> => {
try {
return await cacheWrapper(event.request, event);
} catch (err: unknown) {
sentry && sentry.captureException(err);
/* workaround for silly TypeScript things */
const error = err as Error;
console.error(error.stack);
return new Response(Strings.ERROR_HTML, {
headers: {
...Constants.RESPONSE_HEADERS,
'content-type': 'text/html',
'cache-control': 'max-age=0, no-cache, no-store, must-revalidate'
},
status: 200
});
}
})()
);
};
/* Event to receive web requests on Cloudflare Worker */
addEventListener('fetch', (event: FetchEvent) => {
sentryWrapper(event);
});