mirror of
https://github.com/CompeyDev/fxtwitter-docker.git
synced 2025-04-04 10:00:55 +01:00
Merge branch 'thread-api' into temp20231030
This commit is contained in:
dangered wolf
commit
0db4036003
GPG key ID:
4AEE18F83AFDEB23
11 changed files with 647 additions and 63 deletions
|
|
@ -61,6 +61,7 @@ const populateTweetProperties = async (
|
|||
name: apiUser.name,
|
||||
screen_name: apiUser.screen_name,
|
||||
avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '',
|
||||
// @ts-expect-error Legacy api shit
|
||||
avatar_color: null,
|
||||
banner_url: apiUser.banner_url || '',
|
||||
description: apiUser.description || '',
|
||||
|
|
@ -77,8 +78,11 @@ const populateTweetProperties = async (
|
|||
};
|
||||
apiTweet.replies = tweet.legacy.reply_count;
|
||||
apiTweet.retweets = tweet.legacy.retweet_count;
|
||||
apiTweet.reposts = tweet.legacy.retweet_count;
|
||||
apiTweet.likes = tweet.legacy.favorite_count;
|
||||
// @ts-expect-error Legacy api shit
|
||||
apiTweet.color = null;
|
||||
// @ts-expect-error legacy api
|
||||
apiTweet.twitter_card = 'tweet';
|
||||
apiTweet.created_at = tweet.legacy.created_at;
|
||||
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
|
||||
|
|
@ -131,11 +135,11 @@ const populateTweetProperties = async (
|
|||
apiTweet.media.all.push(mediaObject);
|
||||
|
||||
if (mediaObject.type === 'photo') {
|
||||
apiTweet.twitter_card = 'summary_large_image';
|
||||
apiTweet.embed_card = 'summary_large_image';
|
||||
apiTweet.media.photos = apiTweet.media.photos || [];
|
||||
apiTweet.media.photos.push(mediaObject);
|
||||
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
|
||||
apiTweet.twitter_card = 'player';
|
||||
apiTweet.embed_card = 'player';
|
||||
apiTweet.media.videos = apiTweet.media.videos || [];
|
||||
apiTweet.media.videos.push(mediaObject);
|
||||
} else {
|
||||
|
|
@ -183,9 +187,9 @@ const populateTweetProperties = async (
|
|||
/* Workaround: Force player card by default for videos */
|
||||
/* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
|
||||
The mediaList however can set it to something else. TODO: Reimplement as enums */
|
||||
// @ts-expect-error see above comment
|
||||
if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') {
|
||||
apiTweet.twitter_card = 'player';
|
||||
|
||||
if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
|
||||
apiTweet.embed_card = 'player';
|
||||
}
|
||||
|
||||
/* If a language is specified in API or by user, let's try translating it! */
|
||||
|
|
@ -297,8 +301,8 @@ export const statusAPI = async (
|
|||
if (quoteTweet) {
|
||||
apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet;
|
||||
/* Only override the twitter_card if it's a basic tweet, since media always takes precedence */
|
||||
if (apiTweet.twitter_card === 'tweet' && apiTweet.quote !== null) {
|
||||
apiTweet.twitter_card = apiTweet.quote.twitter_card;
|
||||
if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) {
|
||||
apiTweet.embed_card = apiTweet.quote.embed_card;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
|
|||
apiUser.following = user.legacy.friends_count;
|
||||
apiUser.likes = user.legacy.favourites_count;
|
||||
apiUser.tweets = user.legacy.statuses_count;
|
||||
apiUser.posts = user.legacy.statuses_count;
|
||||
apiUser.name = user.legacy.name;
|
||||
apiUser.screen_name = user.legacy.screen_name;
|
||||
apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description);
|
||||
apiUser.location = user.legacy.location;
|
||||
apiUser.banner_url = user.legacy.profile_banner_url;
|
||||
apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`
|
||||
apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null;
|
||||
apiUser.location = user.legacy.location ? user.legacy.location : null;
|
||||
apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null;
|
||||
/*
|
||||
if (user.is_blue_verified) {
|
||||
apiUser.verified = 'blue';
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ export const Constants = {
|
|||
RELEASE_NAME: RELEASE_NAME,
|
||||
API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`,
|
||||
TWITTER_ROOT: 'https://twitter.com',
|
||||
TWITTER_GLOBAL_NAME_ROOT: 'twitter.com',
|
||||
TWITTER_API_ROOT: 'https://api.twitter.com',
|
||||
BOT_UA_REGEX:
|
||||
/bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi,
|
||||
|
|
@ -58,8 +59,8 @@ export const Constants = {
|
|||
RESPONSE_HEADERS: {
|
||||
'allow': 'OPTIONS, GET, PURGE, HEAD',
|
||||
'content-type': 'text/html;charset=UTF-8',
|
||||
'x-powered-by': '🏳️⚧️ Trans Rights',
|
||||
'cache-control': 'max-age=3600' // Can be overriden in some cases, like poll tweets
|
||||
'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`,
|
||||
'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets
|
||||
},
|
||||
API_RESPONSE_HEADERS: {
|
||||
'access-control-allow-origin': '*',
|
||||
|
|
@ -68,8 +69,3 @@ export const Constants = {
|
|||
POLL_TWEET_CACHE: 'max-age=60',
|
||||
DEFAULT_COLOR: '#10A3FF'
|
||||
};
|
||||
|
||||
if (typeof TEST !== 'undefined') {
|
||||
/* Undici gets angry about unicode headers, this is a workaround. */
|
||||
Constants.RESPONSE_HEADERS['x-powered-by'] = 'Trans Rights';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ export const handleStatus = async (
|
|||
const headers = [
|
||||
`<link rel="canonical" href="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
|
||||
`<meta property="og:url" content="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
|
||||
`<meta property="theme-color" content="${tweet.color || '#00a8fc'}"/>`,
|
||||
`<meta property="theme-color" content="#00a8fc"/>`,
|
||||
`<meta property="twitter:site" content="@${tweet.author.screen_name}"/>`,
|
||||
`<meta property="twitter:creator" content="@${tweet.author.screen_name}"/>`,
|
||||
`<meta property="twitter:title" content="${tweet.author.name} (@${tweet.author.screen_name})"/>`
|
||||
|
|
@ -200,8 +200,8 @@ export const handleStatus = async (
|
|||
siteName = instructions.siteName;
|
||||
}
|
||||
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
|
||||
if (tweet.twitter_card === 'player') {
|
||||
tweet.twitter_card = 'summary_large_image';
|
||||
if (tweet.embed_card === 'player') {
|
||||
tweet.embed_card = 'summary_large_image';
|
||||
}
|
||||
break;
|
||||
case 'video':
|
||||
|
|
@ -217,8 +217,8 @@ export const handleStatus = async (
|
|||
siteName = instructions.siteName;
|
||||
}
|
||||
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
|
||||
if (tweet.twitter_card !== 'player') {
|
||||
tweet.twitter_card = 'player';
|
||||
if (tweet.embed_card !== 'player') {
|
||||
tweet.embed_card = 'player';
|
||||
}
|
||||
/* This Tweet has a video to render. */
|
||||
break;
|
||||
|
|
@ -345,7 +345,7 @@ export const handleStatus = async (
|
|||
and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */
|
||||
const text = useIV ? sanitizeText(newText).replace(/\n/g, '<br>') : sanitizeText(newText);
|
||||
|
||||
const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card;
|
||||
const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card;
|
||||
|
||||
/* Push basic headers relating to author, Tweet text, and site name */
|
||||
headers.push(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { Strings } from '../strings';
|
||||
|
||||
/* Helper for Quote Tweets */
|
||||
export const handleQuote = (quote: APITweet): string | null => {
|
||||
export const handleQuote = (quote: APIPost): string | null => {
|
||||
console.log('Quoting status ', quote.id);
|
||||
|
||||
let str = `\n`;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
/* We keep this value up-to-date for making our requests to Twitter as
|
||||
indistinguishable from normal user traffic as possible. */
|
||||
const fakeChromeVersion = 116;
|
||||
const fakeChromeVersion = 118;
|
||||
const platformWindows = 'Windows NT 10.0; Win64; x64';
|
||||
const platformMac = 'Macintosh; Intel Mac OS X 10_15_7';
|
||||
const platformLinux = 'X11; Linux x86_64';
|
||||
|
|
|
|||
223
src/providers/twitter/processor.ts
Normal file
223
src/providers/twitter/processor.ts
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
import { renderCard } from '../../helpers/card';
|
||||
import { Constants } from '../../constants';
|
||||
import { linkFixer } from '../../helpers/linkFixer';
|
||||
import { handleMosaic } from '../../helpers/mosaic';
|
||||
// import { translateTweet } from '../../helpers/translate';
|
||||
import { unescapeText } from '../../helpers/utils';
|
||||
import { processMedia } from '../../helpers/media';
|
||||
import { convertToApiUser } from '../../api/user';
|
||||
import { translateTweet } from '../../helpers/translate';
|
||||
|
||||
export const buildAPITweet = async (
|
||||
tweet: GraphQLTweet,
|
||||
language: string | undefined,
|
||||
threadPiece = false,
|
||||
legacyAPI = false
|
||||
// eslint-disable-next-line sonarjs/cognitive-complexity
|
||||
): Promise<APITweet | null> => {
|
||||
const apiTweet = {} as APITweet;
|
||||
|
||||
/* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
|
||||
It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */
|
||||
|
||||
if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') {
|
||||
tweet = tweet.result;
|
||||
}
|
||||
|
||||
if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') {
|
||||
tweet.core = tweet.tweet.core;
|
||||
}
|
||||
|
||||
if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') {
|
||||
tweet.legacy = tweet.tweet?.legacy;
|
||||
}
|
||||
|
||||
if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') {
|
||||
tweet.views = tweet?.tweet?.views;
|
||||
}
|
||||
|
||||
if (typeof tweet.core === 'undefined') {
|
||||
console.log('Tweet still not valid', tweet);
|
||||
return null;
|
||||
}
|
||||
|
||||
const graphQLUser = tweet.core.user_results.result;
|
||||
const apiUser = convertToApiUser(graphQLUser);
|
||||
|
||||
/* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */
|
||||
const id = tweet.rest_id ?? tweet.legacy.id_str;
|
||||
|
||||
/* Populating a lot of the basics */
|
||||
apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
|
||||
apiTweet.id = id;
|
||||
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || ''));
|
||||
if (!threadPiece) {
|
||||
apiTweet.author = {
|
||||
id: apiUser.id,
|
||||
name: apiUser.name,
|
||||
screen_name: apiUser.screen_name,
|
||||
global_screen_name: apiUser.global_screen_name,
|
||||
avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null,
|
||||
banner_url: apiUser.banner_url ?? null,
|
||||
description: apiUser.description ?? null,
|
||||
location: apiUser.location ?? null,
|
||||
url: apiUser.url ?? null,
|
||||
followers: apiUser.followers,
|
||||
following: apiUser.following,
|
||||
joined: apiUser.joined,
|
||||
posts: apiUser.tweets,
|
||||
likes: apiUser.likes,
|
||||
protected: apiUser.protected,
|
||||
birthday: apiUser.birthday,
|
||||
website: apiUser.website
|
||||
};
|
||||
}
|
||||
apiTweet.replies = tweet.legacy.reply_count;
|
||||
if (legacyAPI) {
|
||||
apiTweet.retweets = tweet.legacy.retweet_count;
|
||||
} else {
|
||||
apiTweet.reposts = tweet.legacy.retweet_count;
|
||||
}
|
||||
apiTweet.likes = tweet.legacy.favorite_count;
|
||||
apiTweet.embed_card = 'tweet';
|
||||
apiTweet.created_at = tweet.legacy.created_at;
|
||||
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
|
||||
|
||||
apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive;
|
||||
|
||||
if (tweet.views.state === 'EnabledWithCount') {
|
||||
apiTweet.views = parseInt(tweet.views.count || '0') ?? null;
|
||||
} else {
|
||||
apiTweet.views = null;
|
||||
}
|
||||
console.log('note_tweet', JSON.stringify(tweet.note_tweet));
|
||||
const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text;
|
||||
|
||||
if (noteTweetText) {
|
||||
tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls;
|
||||
tweet.legacy.entities.hashtags =
|
||||
tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags;
|
||||
tweet.legacy.entities.symbols =
|
||||
tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols;
|
||||
|
||||
console.log('We meet the conditions to use new note tweets');
|
||||
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText));
|
||||
apiTweet.is_note_tweet = true;
|
||||
} else {
|
||||
apiTweet.is_note_tweet = false;
|
||||
}
|
||||
|
||||
if (tweet.legacy.lang !== 'unk') {
|
||||
apiTweet.lang = tweet.legacy.lang;
|
||||
} else {
|
||||
apiTweet.lang = null;
|
||||
}
|
||||
|
||||
if (legacyAPI) {
|
||||
apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
|
||||
apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
|
||||
} else if (tweet.legacy.in_reply_to_screen_name) {
|
||||
apiTweet.reply_of = {
|
||||
screen_name: tweet.legacy.in_reply_to_screen_name || null,
|
||||
post: tweet.legacy.in_reply_to_status_id_str || null
|
||||
};
|
||||
} else {
|
||||
apiTweet.reply_of = null;
|
||||
}
|
||||
|
||||
apiTweet.media = {
|
||||
all: [],
|
||||
photos: [],
|
||||
videos: [],
|
||||
};
|
||||
|
||||
/* We found a quote tweet, let's process that too */
|
||||
const quoteTweet = tweet.quoted_status_result;
|
||||
if (quoteTweet) {
|
||||
apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet;
|
||||
/* Only override the embed_card if it's a basic tweet, since media always takes precedence */
|
||||
if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) {
|
||||
apiTweet.embed_card = apiTweet.quote.embed_card;
|
||||
}
|
||||
}
|
||||
|
||||
const mediaList = Array.from(
|
||||
tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []
|
||||
);
|
||||
|
||||
// console.log('tweet', JSON.stringify(tweet));
|
||||
|
||||
/* Populate this Tweet's media */
|
||||
mediaList.forEach(media => {
|
||||
const mediaObject = processMedia(media);
|
||||
if (mediaObject) {
|
||||
apiTweet.media?.all?.push(mediaObject);
|
||||
if (mediaObject.type === 'photo') {
|
||||
apiTweet.embed_card = 'summary_large_image';
|
||||
apiTweet.media?.photos?.push(mediaObject);
|
||||
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
|
||||
apiTweet.embed_card = 'player';
|
||||
apiTweet.media?.videos?.push(mediaObject);
|
||||
} else {
|
||||
console.log('Unknown media type', mediaObject.type);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/* Grab color palette data */
|
||||
/*
|
||||
if (mediaList[0]?.ext_media_color?.palette) {
|
||||
apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette);
|
||||
}
|
||||
*/
|
||||
|
||||
/* Handle photos and mosaic if available */
|
||||
if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) {
|
||||
const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
|
||||
if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
|
||||
apiTweet.media.mosaic = mosaic;
|
||||
}
|
||||
}
|
||||
|
||||
// Add Tweet source but remove the link HTML tag
|
||||
if (tweet.source) {
|
||||
apiTweet.source = (tweet.source || '').replace(
|
||||
/<a href="(.+?)" rel="nofollow">(.+?)<\/a>/,
|
||||
'$2'
|
||||
);
|
||||
}
|
||||
|
||||
/* Populate a Twitter card */
|
||||
|
||||
if (tweet.card) {
|
||||
const card = renderCard(tweet.card);
|
||||
if (card.external_media) {
|
||||
apiTweet.media = apiTweet.media ?? {};
|
||||
apiTweet.media.external = card.external_media;
|
||||
}
|
||||
if (card.poll) {
|
||||
apiTweet.poll = card.poll;
|
||||
}
|
||||
}
|
||||
|
||||
if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
|
||||
apiTweet.embed_card = 'player';
|
||||
}
|
||||
|
||||
/* If a language is specified in API or by user, let's try translating it! */
|
||||
if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
|
||||
/* TODO: Reimplement */
|
||||
console.log(`Attempting to translate Tweet to ${language}...`);
|
||||
const translateAPI = await translateTweet(tweet, '', language);
|
||||
if (translateAPI !== null && translateAPI?.translation) {
|
||||
apiTweet.translation = {
|
||||
text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')),
|
||||
source_lang: translateAPI?.sourceLanguage || '',
|
||||
target_lang: translateAPI?.destinationLanguage || '',
|
||||
source_lang_en: translateAPI?.localizedSourceLanguage || ''
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return apiTweet;
|
||||
};
|
||||
313
src/providers/twitter/status.ts
Normal file
313
src/providers/twitter/status.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
import { IRequest } from "itty-router";
|
||||
import { Constants } from "../../constants";
|
||||
import { twitterFetch } from "../../fetch";
|
||||
import { buildAPITweet } from "./processor";
|
||||
|
||||
type GraphQLProcessBucket = {
|
||||
tweets: GraphQLTweet[];
|
||||
cursors: GraphQLTimelineCursor[];
|
||||
}
|
||||
|
||||
type SocialThread = {
|
||||
post: APIPost | APITweet | null;
|
||||
thread: (APIPost | APITweet)[] | null;
|
||||
author: APIUser | null;
|
||||
}
|
||||
|
||||
export const fetchTwitterThread = async (
|
||||
status: string,
|
||||
event: FetchEvent,
|
||||
useElongator = typeof TwitterProxy !== 'undefined',
|
||||
cursor: string | null = null
|
||||
): Promise<GraphQLTweetFoundResponse> => {
|
||||
return (await twitterFetch(
|
||||
`${
|
||||
Constants.TWITTER_ROOT
|
||||
}/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent(
|
||||
JSON.stringify({
|
||||
focalTweetId: status,
|
||||
referrer: "home",
|
||||
with_rux_injections: false,
|
||||
includePromotedContent: false,
|
||||
withCommunity: true,
|
||||
withBirdwatchNotes: true,
|
||||
withQuickPromoteEligibilityTweetFields: false,
|
||||
withVoice: false,
|
||||
withV2Timeline: true,
|
||||
cursor: cursor
|
||||
})
|
||||
)}&features=${encodeURIComponent(
|
||||
JSON.stringify({
|
||||
responsive_web_graphql_exclude_directive_enabled: true,
|
||||
verified_phone_label_enabled: false,
|
||||
responsive_web_home_pinned_timelines_enabled: true,
|
||||
creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
tweetypie_unmention_optimization_enabled: true,
|
||||
responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true,
|
||||
longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: false,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true,
|
||||
standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
|
||||
longform_notetweets_rich_text_read_enabled: true,
|
||||
longform_notetweets_inline_media_enabled: true,
|
||||
responsive_web_media_download_video_enabled: true,
|
||||
responsive_web_enhance_cards_enabled: true
|
||||
})
|
||||
)}&fieldToggles=${encodeURIComponent(
|
||||
JSON.stringify({
|
||||
withArticleRichContentState: true
|
||||
})
|
||||
)}`,
|
||||
event,
|
||||
useElongator,
|
||||
() => {
|
||||
return true;
|
||||
}
|
||||
)) as GraphQLTweetFoundResponse;
|
||||
};
|
||||
|
||||
const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => {
|
||||
const bucket: GraphQLProcessBucket = {
|
||||
tweets: [],
|
||||
cursors: []
|
||||
};
|
||||
instructions.forEach?.(instruction => {
|
||||
if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') {
|
||||
// @ts-expect-error Use entries or moduleItems depending on the type
|
||||
(instruction?.entries ?? instruction.moduleItems).forEach((_entry) => {
|
||||
const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry
|
||||
const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content;
|
||||
if (content.__typename === 'TimelineTimelineItem') {
|
||||
const itemContentType = content.itemContent?.__typename;
|
||||
if (itemContentType === 'TimelineTweet') {
|
||||
const entryType = content.itemContent.tweet_results.result.__typename
|
||||
if (entryType === 'Tweet') {
|
||||
bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet);
|
||||
}
|
||||
if (entryType === 'TweetWithVisibilityResults') {
|
||||
bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet);
|
||||
}
|
||||
} else if (itemContentType === 'TimelineTimelineCursor') {
|
||||
bucket.cursors.push(content.itemContent as GraphQLTimelineCursor);
|
||||
}
|
||||
} else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') {
|
||||
content.items.forEach((item) => {
|
||||
const itemContentType = item.item.itemContent.__typename;
|
||||
if (itemContentType === 'TimelineTweet') {
|
||||
const entryType = item.item.itemContent.tweet_results.result.__typename
|
||||
if (entryType === 'Tweet') {
|
||||
bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet);
|
||||
}
|
||||
if (entryType === 'TweetWithVisibilityResults') {
|
||||
bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet);
|
||||
}
|
||||
} else if (itemContentType === 'TimelineTimelineCursor') {
|
||||
bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
|
||||
return bucket;
|
||||
}
|
||||
|
||||
const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => {
|
||||
return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null;
|
||||
}
|
||||
|
||||
const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => {
|
||||
return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id);
|
||||
}
|
||||
|
||||
const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => {
|
||||
const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id);
|
||||
if (!tweet) {
|
||||
console.log('uhhh, we could not even find that tweet, dunno how that happened');
|
||||
return -1;
|
||||
}
|
||||
const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str);
|
||||
if (index === -1) {
|
||||
console.log('could not find shit for', id)
|
||||
console.log(bucket.cursors)
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => {
|
||||
/* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */
|
||||
return oldCursors.map(cursor => {
|
||||
const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType);
|
||||
if (newCursor) {
|
||||
return newCursor;
|
||||
}
|
||||
return cursor;
|
||||
});
|
||||
}
|
||||
|
||||
const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
|
||||
return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id)
|
||||
}
|
||||
|
||||
export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise<SocialThread> => {
|
||||
const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse;
|
||||
|
||||
if (!response.data) {
|
||||
return { post: null, thread: null, author: null };
|
||||
}
|
||||
|
||||
const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions);
|
||||
const originalTweet = findTweetInBucket(id, bucket);
|
||||
|
||||
/* Don't bother processing thread on a null tweet */
|
||||
if (originalTweet === null) {
|
||||
return { post: null, thread: null, author: null };
|
||||
}
|
||||
|
||||
const post = await buildAPITweet(originalTweet, undefined, false, false);
|
||||
|
||||
if (post === null) {
|
||||
return { post: null, thread: null, author: null };
|
||||
}
|
||||
|
||||
const author = post.author;
|
||||
/* remove post.author */
|
||||
// @ts-expect-error lmao
|
||||
delete post.author;
|
||||
|
||||
/* If we're not processing threads, let's be done here */
|
||||
if (!processThread) {
|
||||
return { post: post, thread: null, author: author };
|
||||
}
|
||||
|
||||
const threadTweets = [originalTweet];
|
||||
bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet);
|
||||
|
||||
let currentId = id;
|
||||
|
||||
/* Process tweets that are following the current one in the thread */
|
||||
while (findNextTweet(currentId, bucket) !== -1) {
|
||||
const index = findNextTweet(currentId, bucket);
|
||||
const tweet = bucket.tweets[index];
|
||||
|
||||
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
|
||||
|
||||
console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
|
||||
|
||||
threadTweets.push(tweet);
|
||||
|
||||
currentId = newCurrentId;
|
||||
|
||||
console.log('Current index', index, 'of', bucket.tweets.length)
|
||||
|
||||
/* Reached the end of the current list of tweets in thread) */
|
||||
if (index >= (bucket.tweets.length - 1)) {
|
||||
/* See if we have a cursor to fetch more tweets */
|
||||
const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore'));
|
||||
console.log('current cursors: ', bucket.cursors)
|
||||
if (!cursor) {
|
||||
console.log('No cursor present, stopping pagination down')
|
||||
break;
|
||||
}
|
||||
console.log('Cursor present, fetching more tweets down');
|
||||
|
||||
let loadCursor: GraphQLTweetFoundResponse;
|
||||
|
||||
try {
|
||||
loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
|
||||
|
||||
if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
|
||||
console.log('Unknown data while fetching cursor', loadCursor);
|
||||
break;
|
||||
}
|
||||
} catch(e) {
|
||||
console.log('Error fetching cursor', e);
|
||||
break;
|
||||
}
|
||||
|
||||
const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
|
||||
bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet));
|
||||
/* Remove old cursor and add new bottom cursor if necessary */
|
||||
consolidateCursors(bucket.cursors, cursorResponse.cursors);
|
||||
console.log('updated bucket of cursors', bucket.cursors);
|
||||
}
|
||||
|
||||
console.log('Preview of next tweet:', findNextTweet(currentId, bucket));
|
||||
}
|
||||
|
||||
currentId = id;
|
||||
|
||||
while (findPreviousTweet(currentId, bucket) !== -1) {
|
||||
const index = findPreviousTweet(currentId, bucket);
|
||||
const tweet = bucket.tweets[index];
|
||||
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
|
||||
|
||||
console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
|
||||
|
||||
threadTweets.unshift(tweet);
|
||||
|
||||
currentId = newCurrentId;
|
||||
|
||||
if (index === 0) {
|
||||
/* See if we have a cursor to fetch more tweets */
|
||||
const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore'));
|
||||
console.log('current cursors: ', bucket.cursors)
|
||||
if (!cursor) {
|
||||
console.log('No cursor present, stopping pagination up')
|
||||
break;
|
||||
}
|
||||
console.log('Cursor present, fetching more tweets up');
|
||||
let loadCursor: GraphQLTweetFoundResponse;
|
||||
|
||||
try {
|
||||
loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
|
||||
|
||||
if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
|
||||
console.log('Unknown data while fetching cursor', loadCursor);
|
||||
break;
|
||||
}
|
||||
} catch(e) {
|
||||
console.log('Error fetching cursor', e);
|
||||
break;
|
||||
}
|
||||
const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
|
||||
bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet));
|
||||
/* Remove old cursor and add new top cursor if necessary */
|
||||
consolidateCursors(bucket.cursors, cursorResponse.cursors);
|
||||
|
||||
// console.log('updated bucket of tweets', bucket.tweets);
|
||||
console.log('updated bucket of cursors', bucket.cursors);
|
||||
}
|
||||
|
||||
console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket));
|
||||
}
|
||||
|
||||
const socialThread: SocialThread = {
|
||||
post: post,
|
||||
thread: [],
|
||||
author: author
|
||||
}
|
||||
|
||||
threadTweets.forEach(async (tweet) => {
|
||||
socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false));
|
||||
});
|
||||
|
||||
return socialThread;
|
||||
}
|
||||
|
||||
export const threadAPIProvider = async (request: IRequest) => {
|
||||
const { id } = request.params;
|
||||
|
||||
const processedResponse = await processTwitterThread(id, true, request);
|
||||
|
||||
return new Response(JSON.stringify(processedResponse), {
|
||||
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }
|
||||
})
|
||||
}
|
||||
69
src/types/twitterTypes.d.ts
vendored
69
src/types/twitterTypes.d.ts
vendored
|
|
@ -354,7 +354,7 @@ type GraphQLTweetLegacy = {
|
|||
type GraphQLTweet = {
|
||||
// Workaround
|
||||
result: GraphQLTweet;
|
||||
__typename: 'Tweet' | 'TweetUnavailable';
|
||||
__typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable';
|
||||
rest_id: string; // "1674824189176590336",
|
||||
has_birdwatch_notes: false;
|
||||
core: {
|
||||
|
|
@ -444,37 +444,76 @@ type TweetTombstone = {
|
|||
};
|
||||
};
|
||||
};
|
||||
|
||||
type GraphQLTimelineTweet = {
|
||||
item: 'TimelineTweet';
|
||||
__typename: 'TimelineTweet';
|
||||
tweet_results: {
|
||||
result: GraphQLTweet | TweetTombstone;
|
||||
};
|
||||
}
|
||||
|
||||
type GraphQLTimelineCursor = {
|
||||
cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore';
|
||||
itemType: 'TimelineTimelineCursor';
|
||||
value: string;
|
||||
__typename: 'TimelineTimelineCursor';
|
||||
}
|
||||
|
||||
interface GraphQLBaseTimeline {
|
||||
entryType: string;
|
||||
__typename: string;
|
||||
}
|
||||
|
||||
type GraphQLTimelineItem = GraphQLBaseTimeline & {
|
||||
entryType: 'TimelineTimelineItem';
|
||||
__typename: 'TimelineTimelineItem';
|
||||
itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor;
|
||||
}
|
||||
|
||||
type GraphQLTimelineModule = GraphQLBaseTimeline & {
|
||||
entryType: 'TimelineTimelineModule';
|
||||
__typename: 'TimelineTimelineModule';
|
||||
items: {
|
||||
entryId: `conversationthread-${number}-tweet-${number}`;
|
||||
item: GraphQLTimelineItem
|
||||
}[];
|
||||
}
|
||||
|
||||
type GraphQLTimelineTweetEntry = {
|
||||
/** The entryID contains the tweet ID */
|
||||
entryId: `tweet-${number}`; // "tweet-1674824189176590336"
|
||||
sortIndex: string;
|
||||
content: {
|
||||
entryType: 'TimelineTimelineItem';
|
||||
__typename: 'TimelineTimelineItem';
|
||||
itemContent: {
|
||||
item: 'TimelineTweet';
|
||||
__typename: 'TimelineTweet';
|
||||
tweet_results: {
|
||||
result: GraphQLTweet | TweetTombstone;
|
||||
};
|
||||
};
|
||||
};
|
||||
content: GraphQLTimelineItem;
|
||||
};
|
||||
|
||||
type GraphQLModuleTweetEntry = {
|
||||
/** The entryID contains the tweet ID */
|
||||
sortIndex: string;
|
||||
item: GraphQLTimelineItem | GraphQLTimelineModule;
|
||||
};
|
||||
|
||||
type GraphQLConversationThread = {
|
||||
entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336"
|
||||
sortIndex: string;
|
||||
content: GraphQLTimelineModule;
|
||||
};
|
||||
|
||||
type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown;
|
||||
|
||||
type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction;
|
||||
type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction;
|
||||
|
||||
type TimeLineAddEntriesInstruction = {
|
||||
type TimelineAddEntriesInstruction = {
|
||||
type: 'TimelineAddEntries';
|
||||
entries: GraphQLTimelineEntry[];
|
||||
};
|
||||
|
||||
type TimeLineTerminateTimelineInstruction = {
|
||||
type TimelineAddModulesInstruction = {
|
||||
type: 'TimelineAddToModule';
|
||||
moduleItems: GraphQLTimelineEntry[];
|
||||
};
|
||||
|
||||
type TimelineTerminateTimelineInstruction = {
|
||||
type: 'TimelineTerminateTimeline';
|
||||
direction: 'Top';
|
||||
};
|
||||
|
|
|
|||
51
src/types/types.d.ts
vendored
51
src/types/types.d.ts
vendored
|
|
@ -81,14 +81,6 @@ interface APITranslate {
|
|||
target_lang: string;
|
||||
}
|
||||
|
||||
interface BaseUser {
|
||||
id?: string;
|
||||
name?: string;
|
||||
screen_name?: string;
|
||||
avatar_url?: string;
|
||||
banner_url?: string;
|
||||
}
|
||||
|
||||
interface APIExternalMedia {
|
||||
type: 'video';
|
||||
url: string;
|
||||
|
|
@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia {
|
|||
};
|
||||
}
|
||||
|
||||
interface APITweet {
|
||||
interface APIPost {
|
||||
id: string;
|
||||
url: string;
|
||||
text: string;
|
||||
|
|
@ -144,15 +136,11 @@ interface APITweet {
|
|||
created_timestamp: number;
|
||||
|
||||
likes: number;
|
||||
retweets: number;
|
||||
reposts: number;
|
||||
replies: number;
|
||||
views?: number | null;
|
||||
|
||||
color: string | null;
|
||||
|
||||
quote?: APITweet;
|
||||
quote?: APIPost;
|
||||
poll?: APIPoll;
|
||||
translation?: APITranslate;
|
||||
author: APIUser;
|
||||
|
||||
media?: {
|
||||
|
|
@ -169,24 +157,41 @@ interface APITweet {
|
|||
replying_to: string | null;
|
||||
replying_to_status: string | null;
|
||||
|
||||
source: string;
|
||||
reply_of: {
|
||||
screen_name: string | null;
|
||||
post: string | null;
|
||||
} | null
|
||||
|
||||
is_note_tweet: boolean;
|
||||
source: string | null;
|
||||
|
||||
twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
|
||||
embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
|
||||
}
|
||||
|
||||
interface APIUser extends BaseUser {
|
||||
interface APITweet extends APIPost {
|
||||
retweets: number;
|
||||
views?: number | null;
|
||||
translation?: APITranslate;
|
||||
|
||||
is_note_tweet: boolean;
|
||||
}
|
||||
|
||||
interface APIUser {
|
||||
id: string;
|
||||
name: string;
|
||||
screen_name: string;
|
||||
global_screen_name: string;
|
||||
avatar_url: string | null;
|
||||
banner_url: string | null;
|
||||
// verified: 'legacy' | 'blue'| 'business' | 'government';
|
||||
// verified_label: string;
|
||||
description: string;
|
||||
location: string;
|
||||
description: string | null;
|
||||
location: string | null;
|
||||
url: string;
|
||||
avatar_color?: string | null;
|
||||
protected: boolean;
|
||||
followers: number;
|
||||
following: number;
|
||||
tweets: number;
|
||||
tweets?: number;
|
||||
posts?: number;
|
||||
likes: number;
|
||||
joined: string;
|
||||
website: {
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { Strings } from './strings';
|
|||
import motd from '../motd.json';
|
||||
import { sanitizeText } from './helpers/utils';
|
||||
import { handleProfile } from './user';
|
||||
import { threadAPIProvider } from './providers/twitter/status';
|
||||
|
||||
declare const globalThis: {
|
||||
fetchCompletedTime: number;
|
||||
|
|
@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest);
|
|||
router.get('/status/:id/:language', statusRequest);
|
||||
router.get('/version', versionRequest);
|
||||
router.get('/set_base_redirect', setRedirectRequest);
|
||||
router.get('/v2/twitter/thread/:id', threadAPIProvider)
|
||||
|
||||
/* Oembeds (used by Discord to enhance responses)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue