DevComp/fxtwitter-docker
1
0
Fork 0
mirror of https://github.com/CompeyDev/fxtwitter-docker.git synced 2025-04-04 18:10:56 +01:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #463 from FixTweet/thread-api

Browse source 
TweetDetail endpoint fetching and internal Twitter Threads API
This commit is contained in:
dangered wolf 2023-11-02 05:23:17 -04:00 • committed by GitHub
commit 0c7d43d0cd
Signed by: DevComp
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 1027 additions and 515 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
esbuild.config.mjs
View file

@ -10,7 +10,10 @@ config();
const gitCommit = execSync('git rev-parse --short HEAD').toString().trim();
const gitCommitFull = execSync('git rev-parse HEAD').toString().trim();
const gitUrl = execSync('git remote get-url origin').toString().trim();
const gitBranch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim().replace(/[\\\/]/g, '-');
const gitBranch = execSync('git rev-parse --abbrev-ref HEAD')
.toString()
.trim()
.replace(/[\\\/]/g, '-');
let workerName = 'fixtweet';

311
src/api/status.ts
View file

@ -1,311 +0,0 @@
import { renderCard } from '../helpers/card';
import { Constants } from '../constants';
import { fetchConversation } from '../fetch';
import { linkFixer } from '../helpers/linkFixer';
import { handleMosaic } from '../helpers/mosaic';
import { translateTweet } from '../helpers/translate';
import { unescapeText } from '../helpers/utils';
import { processMedia } from '../helpers/media';
import { convertToApiUser } from './user';
import { isGraphQLTweet } from '../helpers/graphql';
/* This function does the heavy lifting of processing data from Twitter API
and using it to create FixTweet's streamlined API responses */
const populateTweetProperties = async (
tweet: GraphQLTweet,
conversation: TweetResultsByRestIdResult, // TimelineBlobPartial,
language: string | undefined
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<APITweet | null> => {
const apiTweet = {} as APITweet;
/* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */
if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') {
tweet = tweet.result;
}
if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') {
tweet.core = tweet.tweet.core;
}
if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') {
tweet.legacy = tweet.tweet?.legacy;
}
if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') {
tweet.views = tweet?.tweet?.views;
}
if (typeof tweet.core === 'undefined') {
console.log('Tweet still not valid', tweet);
return null;
}
/* With v2 conversation API we re-add the user object ot the tweet because
Twitter stores it separately in the conversation API. This is to consolidate
it in case a user appears multiple times in a thread. */
const graphQLUser = tweet.core.user_results.result;
const apiUser = convertToApiUser(graphQLUser);
/* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */
const id = tweet.rest_id ?? tweet.legacy.id_str;
/* Populating a lot of the basics */
apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
apiTweet.id = id;
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || ''));
apiTweet.author = {
id: apiUser.id,
name: apiUser.name,
screen_name: apiUser.screen_name,
avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '',
avatar_color: null,
banner_url: apiUser.banner_url || '',
description: apiUser.description || '',
location: apiUser.location || '',
url: apiUser.url || '',
followers: apiUser.followers,
following: apiUser.following,
joined: apiUser.joined,
tweets: apiUser.tweets,
likes: apiUser.likes,
protected: apiUser.protected,
birthday: apiUser.birthday,
website: apiUser.website
};
apiTweet.replies = tweet.legacy.reply_count;
apiTweet.retweets = tweet.legacy.retweet_count;
apiTweet.likes = tweet.legacy.favorite_count;
apiTweet.color = null;
apiTweet.twitter_card = 'tweet';
apiTweet.created_at = tweet.legacy.created_at;
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive;
if (tweet.views.state === 'EnabledWithCount') {
apiTweet.views = parseInt(tweet.views.count || '0') ?? null;
} else {
apiTweet.views = null;
}
console.log('note_tweet', JSON.stringify(tweet.note_tweet));
const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text;
if (noteTweetText) {
tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls;
tweet.legacy.entities.hashtags =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags;
tweet.legacy.entities.symbols =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols;
console.log('We meet the conditions to use new note tweets');
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText));
apiTweet.is_note_tweet = true;
} else {
apiTweet.is_note_tweet = false;
}
if (tweet.legacy.lang !== 'unk') {
apiTweet.lang = tweet.legacy.lang;
} else {
apiTweet.lang = null;
}
apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
const mediaList = Array.from(
tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []
);
// console.log('tweet', JSON.stringify(tweet));
/* Populate this Tweet's media */
mediaList.forEach(media => {
const mediaObject = processMedia(media);
if (mediaObject) {
apiTweet.media = apiTweet.media || {};
apiTweet.media.all = apiTweet.media?.all || [];
apiTweet.media.all.push(mediaObject);
if (mediaObject.type === 'photo') {
apiTweet.twitter_card = 'summary_large_image';
apiTweet.media.photos = apiTweet.media.photos || [];
apiTweet.media.photos.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
apiTweet.twitter_card = 'player';
apiTweet.media.videos = apiTweet.media.videos || [];
apiTweet.media.videos.push(mediaObject);
} else {
console.log('Unknown media type', mediaObject.type);
}
}
});
/* Grab color palette data */
/*
if (mediaList[0]?.ext_media_color?.palette) {
apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette);
}
*/
/* Handle photos and mosaic if available */
if ((apiTweet?.media?.photos?.length || 0) > 1) {
const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
apiTweet.media.mosaic = mosaic;
}
}
// Add Tweet source but remove the link HTML tag
if (tweet.source) {
apiTweet.source = (tweet.source || '').replace(
/<a href="(.+?)" rel="nofollow">(.+?)<\/a>/,
'$2'
);
}
/* Populate a Twitter card */
if (tweet.card) {
const card = renderCard(tweet.card);
if (card.external_media) {
apiTweet.media = apiTweet.media || {};
apiTweet.media.external = card.external_media;
}
if (card.poll) {
apiTweet.poll = card.poll;
}
}
/* Workaround: Force player card by default for videos */
/* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
The mediaList however can set it to something else. TODO: Reimplement as enums */
// @ts-expect-error see above comment
if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') {
apiTweet.twitter_card = 'player';
}
/* If a language is specified in API or by user, let's try translating it! */
if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
console.log(`Attempting to translate Tweet to ${language}...`);
const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language);
if (translateAPI !== null && translateAPI?.translation) {
apiTweet.translation = {
text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')),
source_lang: translateAPI?.sourceLanguage || '',
target_lang: translateAPI?.destinationLanguage || '',
source_lang_en: translateAPI?.localizedSourceLanguage || ''
};
}
}
return apiTweet;
};
const writeDataPoint = (
event: FetchEvent,
language: string | undefined,
nsfw: boolean,
returnCode: string,
flags?: InputFlags
) => {
console.log('Writing data point...');
if (typeof AnalyticsEngine !== 'undefined') {
const flagString =
Object.keys(flags || {})
// @ts-expect-error - TypeScript doesn't like iterating over the keys, but that's OK
.filter(flag => flags?.[flag])[0] || 'standard';
AnalyticsEngine.writeDataPoint({
blobs: [
event.request.cf?.colo as string /* Datacenter location */,
event.request.cf?.country as string /* Country code */,
event.request.headers.get('user-agent') ??
'' /* User agent (for aggregating bots calling) */,
returnCode /* Return code */,
flagString /* Type of request */,
language ?? '' /* For translate feature */
],
doubles: [nsfw ? 1 : 0 /* NSFW media = 1, No NSFW Media = 0 */],
indexes: [event.request.headers.get('cf-ray') ?? '' /* CF Ray */]
});
}
};
/* API for Twitter statuses (Tweets)
Used internally by FixTweet's embed service, or
available for free using api.fxtwitter.com. */
export const statusAPI = async (
status: string,
language: string | undefined,
event: FetchEvent,
flags?: InputFlags
): Promise<TweetAPIResponse> => {
const res = await fetchConversation(status, event);
const tweet = res.data?.tweetResult?.result;
if (!tweet) {
return { code: 404, message: 'NOT_FOUND' };
}
/* We're handling this in the actual fetch code now */
// if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
// wasMediaBlockedNSFW = true;
// res = await fetchConversation(status, event, true);
// }
// console.log(JSON.stringify(tweet))
if (tweet.__typename === 'TweetUnavailable') {
if ((tweet as { reason: string })?.reason === 'Protected') {
writeDataPoint(event, language, false, 'PRIVATE_TWEET', flags);
return { code: 401, message: 'PRIVATE_TWEET' };
// } else if (tweet.reason === 'NsfwLoggedOut') {
// // API failure as elongator should have handled this
// writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags);
// return { code: 500, message: 'API_FAIL' };
} else {
// Api failure at parsing status
writeDataPoint(event, language, false, 'API_FAIL', flags);
return { code: 500, message: 'API_FAIL' };
}
}
// If the tweet is not a graphQL tweet something went wrong
if (!isGraphQLTweet(tweet)) {
console.log('Tweet was not a valid tweet', tweet);
writeDataPoint(event, language, false, 'API_FAIL', flags);
return { code: 500, message: 'API_FAIL' };
}
/*
if (tweet.retweeted_status_id_str) {
tweet = conversation?.globalObjects?.tweets?.[tweet.retweeted_status_id_str] || {};
}
*/
if (!tweet) {
return { code: 404, message: 'NOT_FOUND' };
}
/* Creating the response objects */
const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse;
const apiTweet: APITweet = (await populateTweetProperties(tweet, res, language)) as APITweet;
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet;
/* Only override the twitter_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.twitter_card === 'tweet' && apiTweet.quote !== null) {
apiTweet.twitter_card = apiTweet.quote.twitter_card;
}
}
/* Finally, staple the Tweet to the response and return it */
response.tweet = apiTweet;
writeDataPoint(event, language, false, 'OK', flags);
return response;
};

13
src/api/user.ts
View file

@ -10,12 +10,16 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
apiUser.followers = user.legacy.followers_count;
apiUser.following = user.legacy.friends_count;
apiUser.likes = user.legacy.favourites_count;
// @ts-expect-error `tweets` is only part of legacy API
apiUser.tweets = user.legacy.statuses_count;
apiUser.name = user.legacy.name;
apiUser.screen_name = user.legacy.screen_name;
apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description);
apiUser.location = user.legacy.location;
apiUser.banner_url = user.legacy.profile_banner_url;
apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`;
apiUser.description = user.legacy.description
? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description)
: '';
apiUser.location = user.legacy.location ? user.legacy.location : '';
apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : '';
/*
if (user.is_blue_verified) {
apiUser.verified = 'blue';
@ -85,6 +89,9 @@ export const userAPI = async (
const response: UserAPIResponse = { code: 200, message: 'OK' } as UserAPIResponse;
const apiUser: APIUser = (await populateUserProperties(userResponse)) as APIUser;
/* Currently, we haven't rolled this out as it's part of the proto-v2 API */
delete apiUser.global_screen_name;
/* Finally, staple the User to the response and return it */
response.user = apiUser;

10
src/constants.ts
View file

@ -15,6 +15,7 @@ export const Constants = {
RELEASE_NAME: RELEASE_NAME,
API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`,
TWITTER_ROOT: 'https://twitter.com',
TWITTER_GLOBAL_NAME_ROOT: 'twitter.com',
TWITTER_API_ROOT: 'https://api.twitter.com',
BOT_UA_REGEX:
/bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi,
@ -58,8 +59,8 @@ export const Constants = {
RESPONSE_HEADERS: {
'allow': 'OPTIONS, GET, PURGE, HEAD',
'content-type': 'text/html;charset=UTF-8',
'x-powered-by': '🏳️‍⚧️ Trans Rights',
'cache-control': 'max-age=3600' // Can be overriden in some cases, like poll tweets
'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`,
'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets
},
API_RESPONSE_HEADERS: {
'access-control-allow-origin': '*',
@ -68,8 +69,3 @@ export const Constants = {
POLL_TWEET_CACHE: 'max-age=60',
DEFAULT_COLOR: '#10A3FF'
};
if (typeof TEST !== 'undefined') {
/* Undici gets angry about unicode headers, this is a workaround. */
Constants.RESPONSE_HEADERS['x-powered-by'] = 'Trans Rights';
}

73
src/embed/status.ts
View file

@ -3,10 +3,11 @@ import { handleQuote } from '../helpers/quote';
import { formatNumber, sanitizeText, truncateWithEllipsis } from '../helpers/utils';
import { Strings } from '../strings';
import { getAuthorText } from '../helpers/author';
import { statusAPI } from '../api/status';
import { renderPhoto } from '../render/photo';
import { renderVideo } from '../render/video';
import { renderInstantView } from '../render/instantview';
import { constructTwitterThread } from '../providers/twitter/conversation';
import { IRequest } from 'itty-router';
export const returnError = (error: string): StatusResponse => {
return {
@ -24,17 +25,55 @@ export const returnError = (error: string): StatusResponse => {
Like Twitter, we use the terminologies interchangably. */
export const handleStatus = async (
status: string,
mediaNumber?: number,
userAgent?: string,
flags?: InputFlags,
language?: string,
event?: FetchEvent
mediaNumber: number | undefined,
userAgent: string,
flags: InputFlags,
language: string,
event: FetchEvent,
request: IRequest
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<StatusResponse> => {
console.log('Direct?', flags?.direct);
const api = await statusAPI(status, language, event as FetchEvent, flags);
const tweet = api?.tweet as APITweet;
console.log('event', event)
let fetchWithThreads = false;
if (request?.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) {
fetchWithThreads = true;
}
const thread = await constructTwitterThread(
status,
fetchWithThreads,
request,
language,
flags?.api ?? false
);
const tweet = thread?.post as APITweet;
const api = {
code: thread.code,
message: '',
tweet: tweet
};
switch (api.code) {
case 200:
api.message = 'OK';
break;
case 401:
api.message = 'PRIVATE_TWEET';
break;
case 404:
api.message = 'NOT_FOUND';
break;
case 500:
console.log(api);
api.message = 'API_FAIL';
break;
}
/* Catch this request if it's an API response */
if (flags?.api) {
@ -46,6 +85,10 @@ export const handleStatus = async (
};
}
if (tweet === null) {
return returnError(Strings.ERROR_TWEET_NOT_FOUND);
}
/* If there was any errors fetching the Tweet, we'll return it */
switch (api.code) {
case 401:
@ -121,7 +164,7 @@ export const handleStatus = async (
const headers = [
`<link rel="canonical" href="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
`<meta property="og:url" content="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
`<meta property="theme-color" content="${tweet.color || '#00a8fc'}"/>`,
`<meta property="theme-color" content="#00a8fc"/>`,
`<meta property="twitter:site" content="@${tweet.author.screen_name}"/>`,
`<meta property="twitter:creator" content="@${tweet.author.screen_name}"/>`,
`<meta property="twitter:title" content="${tweet.author.name} (@${tweet.author.screen_name})"/>`
@ -200,8 +243,8 @@ export const handleStatus = async (
siteName = instructions.siteName;
}
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
if (tweet.twitter_card === 'player') {
tweet.twitter_card = 'summary_large_image';
if (tweet.embed_card === 'player') {
tweet.embed_card = 'summary_large_image';
}
break;
case 'video':
@ -217,8 +260,8 @@ export const handleStatus = async (
siteName = instructions.siteName;
}
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
if (tweet.twitter_card !== 'player') {
tweet.twitter_card = 'player';
if (tweet.embed_card !== 'player') {
tweet.embed_card = 'player';
}
/* This Tweet has a video to render. */
break;
@ -345,7 +388,7 @@ export const handleStatus = async (
and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */
const text = useIV ? sanitizeText(newText).replace(/\n/g, '<br>') : sanitizeText(newText);
const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card;
const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card;
/* Push basic headers relating to author, Tweet text, and site name */
headers.push(
@ -360,7 +403,7 @@ export const handleStatus = async (
authorText = `↪ Replying to @${tweet.replying_to}`;
/* We'll assume it's a thread if it's a reply to themselves */
} else if (
tweet.replying_to === tweet.author.screen_name &&
tweet.replying_to?.screen_name === tweet.author.screen_name &&
authorText === Strings.DEFAULT_AUTHOR_TEXT
) {
authorText = `↪ A part of @${tweet.author.screen_name}'s thread`;

10
src/experiments.ts
View file

@ -1,6 +1,7 @@
export enum Experiment {
ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT',
ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API'
ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API',
TWEET_DETAIL_API = 'TWEET_DETAIL_API'
}
type ExperimentConfig = {
@ -13,12 +14,17 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = {
[Experiment.ELONGATOR_BY_DEFAULT]: {
name: 'Elongator by default',
description: 'Enable Elongator by default (guest token lockout bypass)',
percentage: 0.6
percentage: 1
},
[Experiment.ELONGATOR_PROFILE_API]: {
name: 'Elongator profile API',
description: 'Use Elongator to load profiles',
percentage: 0
},
[Experiment.TWEET_DETAIL_API]: {
name: 'Tweet detail API',
description: 'Use Tweet Detail API (where available with elongator)',
percentage: 0.75
}
};

98
src/fetch.ts
View file

@ -1,7 +1,6 @@
import { Constants } from './constants';
import { Experiment, experimentCheck } from './experiments';
import { generateUserAgent } from './helpers/useragent';
import { isGraphQLTweet } from './helpers/graphql';
const API_ATTEMPTS = 3;
let wasElongatorDisabled = false;
@ -26,7 +25,8 @@ export const twitterFetch = async (
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
),
validateFunction: (response: unknown) => boolean
validateFunction: (response: unknown) => boolean,
elongatorRequired = false
): Promise<unknown> => {
let apiAttempts = 0;
let newTokenGenerated = false;
@ -163,6 +163,11 @@ export const twitterFetch = async (
/* We'll usually only hit this if we get an invalid response from Twitter.
It's uncommon, but it happens */
console.error('Unknown error while fetching from API', e);
/* Elongator returns strings to communicate downstream errors */
if (String(e).indexOf('Status not found')) {
console.log('Tweet was not found');
return {};
}
!useElongator &&
event &&
event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }));
@ -181,7 +186,6 @@ export const twitterFetch = async (
!wasElongatorDisabled &&
!useElongator &&
typeof TwitterProxy !== 'undefined' &&
// @ts-expect-error This is safe due to optional chaining
(response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason ===
'NsfwLoggedOut'
) {
@ -201,6 +205,10 @@ export const twitterFetch = async (
if (!validateFunction(response)) {
console.log('Failed to fetch response, got', JSON.stringify(response));
if (elongatorRequired) {
console.log('Elongator was required, but we failed to fetch a valid response');
return {};
}
if (useElongator) {
console.log('Elongator request failed to validate, trying again without it');
wasElongatorDisabled = true;
@ -232,87 +240,6 @@ export const twitterFetch = async (
return {};
};
export const fetchConversation = async (
status: string,
event: FetchEvent,
useElongator = experimentCheck(
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
)
): Promise<TweetResultsByRestIdResult> => {
return (await twitterFetch(
`${
Constants.TWITTER_ROOT
}/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent(
JSON.stringify({
tweetId: status,
withCommunity: false,
includePromotedContent: false,
withVoice: false
})
)}&features=${encodeURIComponent(
JSON.stringify({
creator_subscriptions_tweet_preview_api_enabled: true,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
longform_notetweets_rich_text_read_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
responsive_web_media_download_video_enabled: false,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_enhance_cards_enabled: false
})
)}&fieldToggles=${encodeURIComponent(
JSON.stringify({
withArticleRichContentState: true
})
)}`,
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as TweetResultsByRestIdResult;
// If we get a not found error it's still a valid response
const tweet = conversation.data?.tweetResult?.result;
if (isGraphQLTweet(tweet)) {
return true;
}
console.log('invalid graphql tweet');
if (
!tweet &&
typeof conversation.data?.tweetResult === 'object' &&
Object.keys(conversation.data?.tweetResult || {}).length === 0
) {
console.log('tweet was not found');
return true;
}
if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
console.log('tweet is nsfw');
return true;
}
if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
console.log('tweet is protected');
return true;
}
if (tweet?.__typename === 'TweetUnavailable') {
console.log('generic tweet unavailable error');
return true;
}
// Final clause for checking if it's valid is if there's errors
return Array.isArray(conversation.errors);
}
)) as TweetResultsByRestIdResult;
};
export const fetchUser = async (
username: string,
event: FetchEvent,
@ -359,6 +286,7 @@ export const fetchUser = async (
conversation.errors?.[0]?.code === 239)
);
*/
}
},
false
)) as GraphQLUserResponse;
};

12
src/helpers/author.ts
View file

@ -3,13 +3,13 @@ import { formatNumber } from './utils';
/* The embed "author" text we populate with replies, retweets, and likes unless it's a video */
export const getAuthorText = (tweet: APITweet): string | null => {
/* Build out reply, retweet, like counts */
if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) {
if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) {
let authorText = '';
if (tweet.replies > 0) {
authorText += `${formatNumber(tweet.replies)} 💬 `;
}
if (tweet.retweets > 0) {
authorText += `${formatNumber(tweet.retweets)} 🔁 `;
if (tweet.reposts > 0) {
authorText += `${formatNumber(tweet.reposts)} 🔁 `;
}
if (tweet.likes > 0) {
authorText += `${formatNumber(tweet.likes)} ❤️ `;
@ -28,13 +28,13 @@ export const getAuthorText = (tweet: APITweet): string | null => {
/* The embed "author" text we populate with replies, retweets, and likes unless it's a video */
export const getSocialTextIV = (tweet: APITweet): string | null => {
/* Build out reply, retweet, like counts */
if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) {
if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) {
let authorText = '';
if (tweet.replies > 0) {
authorText += `💬 ${formatNumber(tweet.replies)} `;
}
if (tweet.retweets > 0) {
authorText += `🔁 ${formatNumber(tweet.retweets)} `;
if (tweet.reposts > 0) {
authorText += `🔁 ${formatNumber(tweet.reposts)} `;
}
if (tweet.likes > 0) {
authorText += `❤️ ${formatNumber(tweet.likes)} `;

2
src/helpers/mosaic.ts
View file

@ -13,7 +13,7 @@ const getDomain = (twitterId: string): string | null => {
hash = (hash << 5) - hash + char;
}
return mosaicDomains[Math.abs(hash) % mosaicDomains.length];
}
};
/* Handler for mosaic (multi-image combiner) */
export const handleMosaic = async (

2
src/helpers/quote.ts
View file

@ -1,7 +1,7 @@
import { Strings } from '../strings';
/* Helper for Quote Tweets */
export const handleQuote = (quote: APITweet): string | null => {
export const handleQuote = (quote: APIPost): string | null => {
console.log('Quoting status ', quote.id);
let str = `\n`;

2
src/helpers/useragent.ts
View file

@ -1,6 +1,6 @@
/* We keep this value up-to-date for making our requests to Twitter as
indistinguishable from normal user traffic as possible. */
const fakeChromeVersion = 116;
const fakeChromeVersion = 118;
const platformWindows = 'Windows NT 10.0; Win64; x64';
const platformMac = 'Macintosh; Intel Mac OS X 10_15_7';
const platformLinux = 'X11; Linux x86_64';

513
src/providers/twitter/conversation.ts Normal file
View file

@ -0,0 +1,513 @@
import { IRequest } from 'itty-router';
import { Constants } from '../../constants';
import { twitterFetch } from '../../fetch';
import { buildAPITweet } from './processor';
import { Experiment, experimentCheck } from '../../experiments';
import { isGraphQLTweet } from '../../helpers/graphql';
export const fetchTweetDetail = async (
status: string,
event: FetchEvent,
useElongator = typeof TwitterProxy !== 'undefined',
cursor: string | null = null
): Promise<TweetDetailResult> => {
return (await twitterFetch(
`${
Constants.TWITTER_ROOT
}/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent(
JSON.stringify({
focalTweetId: status,
referrer: 'home',
with_rux_injections: false,
includePromotedContent: false,
withCommunity: true,
withBirdwatchNotes: true,
withQuickPromoteEligibilityTweetFields: false,
withVoice: false,
withV2Timeline: true,
cursor: cursor
})
)}&features=${encodeURIComponent(
JSON.stringify({
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
responsive_web_home_pinned_timelines_enabled: true,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
longform_notetweets_rich_text_read_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_media_download_video_enabled: true,
responsive_web_enhance_cards_enabled: true
})
)}&fieldToggles=${encodeURIComponent(
JSON.stringify({
withArticleRichContentState: true
})
)}`,
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as TweetDetailResult;
const tweet = findTweetInBucket(
status,
processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions)
);
if (tweet && isGraphQLTweet(tweet)) {
return true;
}
console.log('invalid graphql tweet', conversation);
const firstInstruction = (
conversation.data?.threaded_conversation_with_injections_v2
.instructions?.[0] as TimelineAddEntriesInstruction
)?.entries?.[0];
if (
(
(firstInstruction as { content: GraphQLTimelineItem })?.content
?.itemContent as GraphQLTimelineTweet
)?.tweet_results?.result?.__typename === 'TweetTombstone'
) {
console.log('tweet is private');
return true;
}
return Array.isArray(conversation?.errors);
},
true
)) as TweetDetailResult;
};
export const fetchByRestId = async (
status: string,
event: FetchEvent,
useElongator = experimentCheck(
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
)
): Promise<TweetResultsByRestIdResult> => {
return (await twitterFetch(
`${
Constants.TWITTER_ROOT
}/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent(
JSON.stringify({
tweetId: status,
withCommunity: false,
includePromotedContent: false,
withVoice: false
})
)}&features=${encodeURIComponent(
JSON.stringify({
creator_subscriptions_tweet_preview_api_enabled: true,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
longform_notetweets_rich_text_read_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
responsive_web_media_download_video_enabled: false,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_enhance_cards_enabled: false
})
)}&fieldToggles=${encodeURIComponent(
JSON.stringify({
withArticleRichContentState: true
})
)}`,
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as TweetResultsByRestIdResult;
// If we get a not found error it's still a valid response
const tweet = conversation.data?.tweetResult?.result;
if (isGraphQLTweet(tweet)) {
return true;
}
console.log('invalid graphql tweet');
if (
!tweet &&
typeof conversation.data?.tweetResult === 'object' &&
Object.keys(conversation.data?.tweetResult || {}).length === 0
) {
console.log('tweet was not found');
return true;
}
if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
console.log('tweet is nsfw');
return true;
}
if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
console.log('tweet is protected');
return true;
}
if (tweet?.__typename === 'TweetUnavailable') {
console.log('generic tweet unavailable error');
return true;
}
// Final clause for checking if it's valid is if there's errors
return Array.isArray(conversation.errors);
},
false
)) as TweetResultsByRestIdResult;
};
const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => {
const bucket: GraphQLProcessBucket = {
tweets: [],
cursors: []
};
instructions.forEach?.(instruction => {
if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') {
// @ts-expect-error Use entries or moduleItems depending on the type
(instruction?.entries ?? instruction.moduleItems).forEach(_entry => {
const entry = _entry as
| GraphQLTimelineTweetEntry
| GraphQLConversationThread
| GraphQLModuleTweetEntry;
const content =
(entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content;
if (content.__typename === 'TimelineTimelineItem') {
const itemContentType = content.itemContent?.__typename;
if (itemContentType === 'TimelineTweet') {
const entryType = content.itemContent.tweet_results.result.__typename;
if (entryType === 'Tweet') {
bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet);
}
if (entryType === 'TweetWithVisibilityResults') {
bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet);
}
} else if (itemContentType === 'TimelineTimelineCursor') {
bucket.cursors.push(content.itemContent as GraphQLTimelineCursor);
}
} else if (
(content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule'
) {
content.items.forEach(item => {
const itemContentType = item.item.itemContent.__typename;
if (itemContentType === 'TimelineTweet') {
const entryType = item.item.itemContent.tweet_results.result.__typename;
if (entryType === 'Tweet') {
bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet);
}
if (entryType === 'TweetWithVisibilityResults') {
bucket.tweets.push(
item.item.itemContent.tweet_results.result.tweet as GraphQLTweet
);
}
} else if (itemContentType === 'TimelineTimelineCursor') {
bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor);
}
});
}
});
}
});
return bucket;
};
const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => {
return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null;
};
const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => {
return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id);
};
const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => {
const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id);
if (!tweet) {
console.log('uhhh, we could not even find that tweet, dunno how that happened');
return -1;
}
return bucket.tweets.findIndex(
_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str
);
};
const consolidateCursors = (
oldCursors: GraphQLTimelineCursor[],
newCursors: GraphQLTimelineCursor[]
): GraphQLTimelineCursor[] => {
/* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */
return oldCursors.map(cursor => {
const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType);
if (newCursor) {
return newCursor;
}
return cursor;
});
};
const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
return tweets.filter(
tweet =>
tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id
);
};
/* Fetch and construct a Twitter thread */
export const constructTwitterThread = async (
id: string,
processThread = false,
request: IRequest,
language: string | undefined,
legacyAPI = false
): Promise<SocialThread> => {
console.log('language', language);
let response: TweetDetailResult | TweetResultsByRestIdResult | null = null;
let post: APITweet;
/* We can use TweetDetail on elongator accounts to increase per-account rate limit.
We also use TweetDetail to process threads (WIP)
Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why.
I'll figure out why eventually, but for now just don't use TweetDetail for this. */
if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) {
console.log('Using TweetDetail for request...');
response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult;
console.log(response);
const firstInstruction = (
response.data?.threaded_conversation_with_injections_v2
.instructions?.[0] as TimelineAddEntriesInstruction
)?.entries?.[0];
if (
(
(firstInstruction as { content: GraphQLTimelineItem })?.content
?.itemContent as GraphQLTimelineTweet
)?.tweet_results?.result?.__typename === 'TweetTombstone' /* If a tweet is private */
) {
console.log('tweet is private');
return { post: null, thread: null, author: null, code: 401 };
} else if (!response.data) {
return { post: null, thread: null, author: null, code: 404 };
}
}
/* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */
if (!response) {
console.log('Using TweetResultsByRestId for request...');
response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult;
const result = response?.data?.tweetResult?.result as GraphQLTweet;
if (typeof result === 'undefined') {
return { post: null, thread: null, author: null, code: 404 };
}
const buildPost = await buildAPITweet(result, language, false, legacyAPI);
if ((buildPost as FetchResults).status === 401) {
return { post: null, thread: null, author: null, code: 401 };
} else if (buildPost === null) {
return { post: null, thread: null, author: null, code: 404 };
}
post = buildPost as APITweet;
return { post: post, thread: null, author: post.author, code: 200 };
}
const bucket = processResponse(
response.data.threaded_conversation_with_injections_v2.instructions
);
const originalTweet = findTweetInBucket(id, bucket);
/* Don't bother processing thread on a null tweet */
if (originalTweet === null) {
return { post: null, thread: null, author: null, code: 404 };
}
post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet;
if (post === null) {
return { post: null, thread: null, author: null, code: 404 };
}
const author = post.author;
/* If we're not processing threads, let's be done here */
if (!processThread) {
return { post: post, thread: null, author: author, code: 200 };
}
const threadTweets = [originalTweet];
bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet);
let currentId = id;
/* Process tweets that are following the current one in the thread */
while (findNextTweet(currentId, bucket) !== -1) {
const index = findNextTweet(currentId, bucket);
const tweet = bucket.tweets[index];
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
console.log(
'adding next tweet to thread',
newCurrentId,
'from',
currentId,
'at index',
index,
'in bucket'
);
threadTweets.push(tweet);
currentId = newCurrentId;
console.log('Current index', index, 'of', bucket.tweets.length);
/* Reached the end of the current list of tweets in thread) */
if (index >= bucket.tweets.length - 1) {
/* See if we have a cursor to fetch more tweets */
const cursor = bucket.cursors.find(
cursor => cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore'
);
console.log('current cursors: ', bucket.cursors);
if (!cursor) {
console.log('No cursor present, stopping pagination down');
break;
}
console.log('Cursor present, fetching more tweets down');
let loadCursor: TweetDetailResult;
try {
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value);
if (
typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions ===
'undefined'
) {
console.log('Unknown data while fetching cursor', loadCursor);
break;
}
} catch (e) {
console.log('Error fetching cursor', e);
break;
}
const cursorResponse = processResponse(
loadCursor.data.threaded_conversation_with_injections_v2.instructions
);
bucket.tweets = bucket.tweets.concat(
filterBucketTweets(cursorResponse.tweets, originalTweet)
);
/* Remove old cursor and add new bottom cursor if necessary */
consolidateCursors(bucket.cursors, cursorResponse.cursors);
console.log('updated bucket of cursors', bucket.cursors);
}
console.log('Preview of next tweet:', findNextTweet(currentId, bucket));
}
currentId = id;
while (findPreviousTweet(currentId, bucket) !== -1) {
const index = findPreviousTweet(currentId, bucket);
const tweet = bucket.tweets[index];
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
console.log(
'adding previous tweet to thread',
newCurrentId,
'from',
currentId,
'at index',
index,
'in bucket'
);
threadTweets.unshift(tweet);
currentId = newCurrentId;
if (index === 0) {
/* See if we have a cursor to fetch more tweets */
const cursor = bucket.cursors.find(
cursor => cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore'
);
console.log('current cursors: ', bucket.cursors);
if (!cursor) {
console.log('No cursor present, stopping pagination up');
break;
}
console.log('Cursor present, fetching more tweets up');
let loadCursor: TweetDetailResult;
try {
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value);
if (
typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions ===
'undefined'
) {
console.log('Unknown data while fetching cursor', loadCursor);
break;
}
} catch (e) {
console.log('Error fetching cursor', e);
break;
}
const cursorResponse = processResponse(
loadCursor.data.threaded_conversation_with_injections_v2.instructions
);
bucket.tweets = cursorResponse.tweets.concat(
filterBucketTweets(bucket.tweets, originalTweet)
);
/* Remove old cursor and add new top cursor if necessary */
consolidateCursors(bucket.cursors, cursorResponse.cursors);
// console.log('updated bucket of tweets', bucket.tweets);
console.log('updated bucket of cursors', bucket.cursors);
}
console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket));
}
const socialThread: SocialThread = {
post: post,
thread: [],
author: author,
code: 200
};
threadTweets.forEach(async tweet => {
socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet);
});
return socialThread;
};
export const threadAPIProvider = async (request: IRequest) => {
const { id } = request.params;
const processedResponse = await constructTwitterThread(id, true, request, undefined);
return new Response(JSON.stringify(processedResponse), {
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }
});
};

265
src/providers/twitter/processor.ts Normal file
View file

@ -0,0 +1,265 @@
import { renderCard } from '../../helpers/card';
import { Constants } from '../../constants';
import { linkFixer } from '../../helpers/linkFixer';
import { handleMosaic } from '../../helpers/mosaic';
// import { translateTweet } from '../../helpers/translate';
import { unescapeText } from '../../helpers/utils';
import { processMedia } from '../../helpers/media';
import { convertToApiUser } from '../../api/user';
import { translateTweet } from '../../helpers/translate';
export const buildAPITweet = async (
tweet: GraphQLTweet,
language: string | undefined,
threadPiece = false,
legacyAPI = false
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<APITweet | FetchResults | null> => {
const apiTweet = {} as APITweet;
/* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */
if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') {
tweet = tweet.result;
}
if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') {
tweet.core = tweet.tweet.core;
}
if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') {
tweet.legacy = tweet.tweet?.legacy;
}
if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') {
tweet.views = tweet?.tweet?.views;
}
if (typeof tweet.core === 'undefined') {
console.log('Tweet still not valid', tweet);
if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
return { status: 401 };
} else {
return { status: 404 };
}
}
const graphQLUser = tweet.core.user_results.result;
const apiUser = convertToApiUser(graphQLUser);
/* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */
const id = tweet.rest_id ?? tweet.legacy.id_str;
/* Populating a lot of the basics */
apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
apiTweet.id = id;
apiTweet.text = unescapeText(
linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')
);
if (!threadPiece) {
apiTweet.author = {
id: apiUser.id,
name: apiUser.name,
screen_name: apiUser.screen_name,
avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null,
banner_url: apiUser.banner_url,
description: apiUser.description,
location: apiUser.location,
url: apiUser.url,
followers: apiUser.followers,
following: apiUser.following,
joined: apiUser.joined,
posts: apiUser.posts,
likes: apiUser.likes,
protected: apiUser.protected,
birthday: apiUser.birthday,
website: apiUser.website
};
}
apiTweet.replies = tweet.legacy.reply_count;
if (legacyAPI) {
// @ts-expect-error Use retweets for legacy API
apiTweet.retweets = tweet.legacy.retweet_count;
// @ts-expect-error `tweets` is only part of legacy API
apiTweet.author.tweets = apiTweet.author.posts;
// @ts-expect-error Part of legacy API that we no longer are able to track
apiTweet.author.avatar_color = null;
// @ts-expect-error Use retweets for legacy API
delete apiTweet.reposts;
// @ts-expect-error Use tweets and not posts for legacy API
delete apiTweet.author.posts;
delete apiTweet.author.global_screen_name;
} else {
apiTweet.reposts = tweet.legacy.retweet_count;
apiTweet.author.global_screen_name = apiUser.global_screen_name;
}
apiTweet.likes = tweet.legacy.favorite_count;
apiTweet.embed_card = 'tweet';
apiTweet.created_at = tweet.legacy.created_at;
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive;
if (tweet.views.state === 'EnabledWithCount') {
apiTweet.views = parseInt(tweet.views.count || '0') ?? null;
} else {
apiTweet.views = null;
}
console.log('note_tweet', JSON.stringify(tweet.note_tweet));
const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text;
if (noteTweetText) {
tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls;
tweet.legacy.entities.hashtags =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags;
tweet.legacy.entities.symbols =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols;
console.log('We meet the conditions to use new note tweets');
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText));
apiTweet.is_note_tweet = true;
} else {
apiTweet.is_note_tweet = false;
}
if (tweet.legacy.lang !== 'unk') {
apiTweet.lang = tweet.legacy.lang;
} else {
apiTweet.lang = null;
}
if (legacyAPI) {
// @ts-expect-error Use replying_to string for legacy API
apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
// @ts-expect-error Use replying_to_status string for legacy API
apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
} else if (tweet.legacy.in_reply_to_screen_name) {
apiTweet.replying_to = {
screen_name: tweet.legacy.in_reply_to_screen_name || null,
post: tweet.legacy.in_reply_to_status_id_str || null
};
} else {
apiTweet.replying_to = null;
}
apiTweet.media = {};
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await buildAPITweet(
quoteTweet,
language,
threadPiece,
legacyAPI
)) as APITweet;
/* Only override the embed_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) {
apiTweet.embed_card = apiTweet.quote.embed_card;
}
}
const mediaList = Array.from(
tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []
);
// console.log('tweet', JSON.stringify(tweet));
/* Populate this Tweet's media */
mediaList.forEach(media => {
const mediaObject = processMedia(media);
if (mediaObject) {
apiTweet.media.all = apiTweet.media?.all ?? [];
apiTweet.media?.all?.push(mediaObject);
if (mediaObject.type === 'photo') {
apiTweet.embed_card = 'summary_large_image';
apiTweet.media.photos = apiTweet.media?.photos ?? [];
apiTweet.media.photos?.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
apiTweet.embed_card = 'player';
apiTweet.media.videos = apiTweet.media?.videos ?? [];
apiTweet.media.videos?.push(mediaObject);
} else {
console.log('Unknown media type', mediaObject.type);
}
}
});
/* Grab color palette data */
/*
if (mediaList[0]?.ext_media_color?.palette) {
apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette);
}
*/
/* Handle photos and mosaic if available */
if ((apiTweet?.media.photos?.length || 0) > 1 && !threadPiece) {
const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
apiTweet.media.mosaic = mosaic;
}
}
// Add Tweet source but remove the link HTML tag
if (tweet.source) {
apiTweet.source = (tweet.source || '').replace(
/<a href="(.+?)" rel="nofollow">(.+?)<\/a>/,
'$2'
);
}
/* Populate a Twitter card */
if (tweet.card) {
const card = renderCard(tweet.card);
if (card.external_media) {
apiTweet.media.external = card.external_media;
}
if (card.poll) {
apiTweet.poll = card.poll;
}
}
if (
apiTweet.media?.videos &&
apiTweet.media?.videos.length > 0 &&
apiTweet.embed_card !== 'player'
) {
apiTweet.embed_card = 'player';
}
console.log('language?', language)
/* If a language is specified in API or by user, let's try translating it! */
if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
console.log(`Attempting to translate Tweet to ${language}...`);
const translateAPI = await translateTweet(tweet, '', language);
if (translateAPI !== null && translateAPI?.translation) {
apiTweet.translation = {
text: unescapeText(
linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')
),
source_lang: translateAPI?.sourceLanguage || '',
target_lang: translateAPI?.destinationLanguage || '',
source_lang_en: translateAPI?.localizedSourceLanguage || ''
};
}
}
if (legacyAPI) {
// @ts-expect-error Use twitter_card for legacy API
apiTweet.twitter_card = apiTweet.embed_card;
// @ts-expect-error Part of legacy API that we no longer are able to track
apiTweet.color = null;
// @ts-expect-error Use twitter_card for legacy API
delete apiTweet.embed_card;
if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) {
// @ts-expect-error media is not required in legacy API if empty
delete apiTweet.media;
}
}
return apiTweet;
};

14
src/render/instantview.ts
View file

@ -4,7 +4,7 @@ import { getSocialTextIV } from '../helpers/author';
import { sanitizeText } from '../helpers/utils';
import { Strings } from '../strings';
const populateUserLinks = (tweet: APITweet, text: string): string => {
const populateUserLinks = (tweet: APIPost, text: string): string => {
/* TODO: Maybe we can add username splices to our API so only genuinely valid users are linked? */
text.match(/@(\w{1,15})/g)?.forEach(match => {
const username = match.replace('@', '');
@ -16,7 +16,7 @@ const populateUserLinks = (tweet: APITweet, text: string): string => {
return text;
};
const generateTweetMedia = (tweet: APITweet): string => {
const generateTweetMedia = (tweet: APIPost): string => {
let media = '';
if (tweet.media?.all?.length) {
tweet.media.all.forEach(mediaItem => {
@ -117,7 +117,7 @@ const truncateSocialCount = (count: number): string => {
}
};
const generateTweetFooter = (tweet: APITweet, isQuote = false): string => {
const generateTweetFooter = (tweet: APIPost, isQuote = false): string => {
const { author } = tweet;
let description = author.description;
@ -131,7 +131,7 @@ const generateTweetFooter = (tweet: APITweet, isQuote = false): string => {
<!-- Embed profile picture, display name, and screen name in table -->
{aboutSection}
`.format({
socialText: getSocialTextIV(tweet) || '',
socialText: getSocialTextIV(tweet as APITweet) || '',
viewOriginal: !isQuote ? `<a href="${tweet.url}">View original post</a>` : notApplicableComment,
aboutSection: isQuote
? ''
@ -156,18 +156,18 @@ const generateTweetFooter = (tweet: APITweet, isQuote = false): string => {
joined: author.joined ? `📆 ${formatDate(new Date(author.joined))}` : '',
following: truncateSocialCount(author.following),
followers: truncateSocialCount(author.followers),
tweets: truncateSocialCount(author.tweets)
tweets: truncateSocialCount(author.posts)
})
});
};
const generateTweet = (tweet: APITweet, isQuote = false): string => {
const generateTweet = (tweet: APIPost, isQuote = false): string => {
let text = paragraphify(sanitizeText(tweet.text), isQuote);
text = htmlifyLinks(text);
text = htmlifyHashtags(text);
text = populateUserLinks(tweet, text);
const translatedText = getTranslatedText(tweet, isQuote);
const translatedText = getTranslatedText(tweet as APITweet, isQuote);
return `<!-- Telegram Instant View -->
{quoteHeader}

2
src/render/photo.ts
View file

@ -35,6 +35,8 @@ export const renderPhoto = (
}
}
console.log('photo!', photo);
if (photo.type === 'mosaic_photo' && !isOverrideMedia) {
instructions.addHeaders = [
`<meta property="twitter:image" content="${tweet.media?.mosaic?.formats.jpeg}"/>`,

95
src/types/twitterTypes.d.ts vendored
View file

@ -354,7 +354,8 @@ type GraphQLTweetLegacy = {
type GraphQLTweet = {
// Workaround
result: GraphQLTweet;
__typename: 'Tweet' | 'TweetUnavailable';
__typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable';
reason: string; // used for errors
rest_id: string; // "1674824189176590336",
has_birdwatch_notes: false;
core: {
@ -444,37 +445,79 @@ type TweetTombstone = {
};
};
};
type GraphQLTimelineTweet = {
item: 'TimelineTweet';
__typename: 'TimelineTweet';
tweet_results: {
result: GraphQLTweet | TweetTombstone;
};
};
type GraphQLTimelineCursor = {
cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore';
itemType: 'TimelineTimelineCursor';
value: string;
__typename: 'TimelineTimelineCursor';
};
interface GraphQLBaseTimeline {
entryType: string;
__typename: string;
}
type GraphQLTimelineItem = GraphQLBaseTimeline & {
entryType: 'TimelineTimelineItem';
__typename: 'TimelineTimelineItem';
itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor;
};
type GraphQLTimelineModule = GraphQLBaseTimeline & {
entryType: 'TimelineTimelineModule';
__typename: 'TimelineTimelineModule';
items: {
entryId: `conversationthread-${number}-tweet-${number}`;
item: GraphQLTimelineItem;
}[];
};
type GraphQLTimelineTweetEntry = {
/** The entryID contains the tweet ID */
entryId: `tweet-${number}`; // "tweet-1674824189176590336"
sortIndex: string;
content: {
entryType: 'TimelineTimelineItem';
__typename: 'TimelineTimelineItem';
itemContent: {
item: 'TimelineTweet';
__typename: 'TimelineTweet';
tweet_results: {
result: GraphQLTweet | TweetTombstone;
};
};
};
content: GraphQLTimelineItem;
};
type GraphQLModuleTweetEntry = {
/** The entryID contains the tweet ID */
sortIndex: string;
item: GraphQLTimelineItem | GraphQLTimelineModule;
};
type GraphQLConversationThread = {
entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336"
sortIndex: string;
content: GraphQLTimelineModule;
};
type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown;
type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction;
type ThreadInstruction =
| TimelineAddEntriesInstruction
| TimelineTerminateTimelineInstruction
| TimelineAddModulesInstruction;
type TimeLineAddEntriesInstruction = {
type TimelineAddEntriesInstruction = {
type: 'TimelineAddEntries';
entries: GraphQLTimelineEntry[];
};
type TimeLineTerminateTimelineInstruction = {
type TimelineAddModulesInstruction = {
type: 'TimelineAddToModule';
moduleItems: GraphQLTimelineEntry[];
};
type TimelineTerminateTimelineInstruction = {
type: 'TimelineTerminateTimeline';
direction: 'Top';
};
@ -504,10 +547,11 @@ type GraphQLTweetNotFoundResponse = {
];
data: Record<string, never>;
};
type GraphQLTweetFoundResponse = {
type TweetDetailResult = {
errors?: unknown[];
data: {
threaded_conversation_with_injections_v2: {
instructions: V2ThreadInstruction[];
instructions: ThreadInstruction[];
};
};
};
@ -516,12 +560,17 @@ type TweetResultsByRestIdResult = {
errors?: unknown[];
data?: {
tweetResult?: {
result?:
| {
__typename: 'TweetUnavailable';
reason: 'NsfwLoggedOut' | 'Protected';
}
| GraphQLTweet;
result?: TweetStub | GraphQLTweet;
};
};
};
type TweetStub = {
__typename: 'TweetUnavailable';
reason: 'NsfwLoggedOut' | 'Protected';
};
interface GraphQLProcessBucket {
tweets: GraphQLTweet[];
cursors: GraphQLTimelineCursor[];
}

91
src/types/types.d.ts vendored
View file

@ -43,31 +43,18 @@ interface Request {
};
}
interface Size {
width: number;
height: number;
}
interface HorizontalSize {
width: number;
height: number;
firstWidth: number;
secondWidth: number;
}
interface VerticalSize {
width: number;
height: number;
firstHeight: number;
secondHeight: number;
}
interface TweetAPIResponse {
code: number;
message: string;
tweet?: APITweet;
}
interface SocialPostAPIResponse {
code: number;
message: string;
post?: APITweet;
}
interface UserAPIResponse {
code: number;
message: string;
@ -81,14 +68,6 @@ interface APITranslate {
target_lang: string;
}
interface BaseUser {
id?: string;
name?: string;
screen_name?: string;
avatar_url?: string;
banner_url?: string;
}
interface APIExternalMedia {
type: 'video';
url: string;
@ -136,7 +115,7 @@ interface APIMosaicPhoto extends APIMedia {
};
}
interface APITweet {
interface APIPost {
id: string;
url: string;
text: string;
@ -144,18 +123,14 @@ interface APITweet {
created_timestamp: number;
likes: number;
retweets: number;
reposts: number;
replies: number;
views?: number | null;
color: string | null;
quote?: APITweet;
quote?: APIPost;
poll?: APIPoll;
translation?: APITranslate;
author: APIUser;
media?: {
media: {
external?: APIExternalMedia;
photos?: APIPhoto[];
videos?: APIVideo[];
@ -166,27 +141,39 @@ interface APITweet {
lang: string | null;
possibly_sensitive: boolean;
replying_to: string | null;
replying_to_status: string | null;
replying_to: {
screen_name: string | null;
post: string | null;
} | null;
source: string;
source: string | null;
is_note_tweet: boolean;
twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
}
interface APIUser extends BaseUser {
interface APITweet extends APIPost {
views?: number | null;
translation?: APITranslate;
is_note_tweet: boolean;
}
interface APIUser {
id: string;
name: string;
screen_name: string;
global_screen_name?: string;
avatar_url: string;
banner_url: string;
// verified: 'legacy' | 'blue'| 'business' | 'government';
// verified_label: string;
description: string;
location: string;
url: string;
avatar_color?: string | null;
protected: boolean;
followers: number;
following: number;
tweets: number;
posts: number;
likes: number;
joined: string;
website: {
@ -199,3 +186,19 @@ interface APIUser extends BaseUser {
year?: number;
};
}
interface SocialPost {
post: APIPost | APITweet | null;
author: APIUser | null;
}
interface SocialThread {
post: APIPost | APITweet | null;
thread: (APIPost | APITweet)[] | null;
author: APIUser | null;
code: number;
}
interface FetchResults {
status: number;
}

7
src/worker.ts
View file

@ -10,6 +10,7 @@ import { Strings } from './strings';
import motd from '../motd.json';
import { sanitizeText } from './helpers/utils';
import { handleProfile } from './user';
// import { threadAPIProvider } from './providers/twitter/conversation';
declare const globalThis: {
fetchCompletedTime: number;
@ -131,6 +132,8 @@ const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputF
console.log('Bypass bot check');
}
console.log('event', event)
/* This throws the necessary data to handleStatus (in status.ts) */
const statusResponse = await handleStatus(
id?.match(/\d{2,20}/)?.[0] || '0',
@ -138,7 +141,8 @@ const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputF
userAgent,
flags,
language,
event
event,
request
);
/* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */
@ -453,6 +457,7 @@ router.get('/status/:id', statusRequest);
router.get('/status/:id/:language', statusRequest);
router.get('/version', versionRequest);
router.get('/set_base_redirect', setRedirectRequest);
// router.get('/v2/twitter/thread/:id', threadAPIProvider)
/* Oembeds (used by Discord to enhance responses)

2
test/index.test.ts
View file

@ -120,8 +120,10 @@ test('API fetch basic Tweet', async () => {
expect(tweet.author.avatar_url).toBeTruthy();
expect(tweet.author.banner_url).toBeTruthy();
expect(tweet.replies).toBeGreaterThan(0);
// @ts-expect-error retweets only in legacy API
expect(tweet.retweets).toBeGreaterThan(0);
expect(tweet.likes).toBeGreaterThan(0);
// @ts-expect-error twitter_card only in legacy API
expect(tweet.twitter_card).toEqual('tweet');
expect(tweet.created_at).toEqual('Tue Mar 21 20:50:14 +0000 2006');
expect(tweet.created_timestamp).toEqual(1142974214);

15
wrangler.example.toml
View file

@ -3,15 +3,16 @@ account_id = "[CLOUDFLARE_ACCOUNT_ID]"
main = "./dist/worker.js"
compatibility_date = "2023-08-15"
send_metrics = false
services = [
{ binding = "TwitterProxy", service = "elongator" }
]
# Remove this if not using Cloudflare Analytics Engine
analytics_engine_datasets = [
{ binding = "AnalyticsEngine" }
]
[build]
command = "npm run build"
# Remove this if not using elongator account proxying
services = [
{ binding = "TwitterProxy", service = "elongator" }
]
[miniflare.globals]
TEST = "true" # Will have unicode character errors in headers if not set to true and running unit tests
[build]
command = "npm run build"