Merge branch 'thread-api' into thread-api-updated

This commit is contained in:
dangered wolf 2023-10-22 14:22:33 -04:00 committed by GitHub
commit 79503844a9
Signed by: DevComp
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 686 additions and 99 deletions

92
package-lock.json generated
View file

@ -1735,9 +1735,9 @@
"dev": true
},
"node_modules/@jridgewell/trace-mapping": {
"version": "0.3.19",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz",
"integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==",
"version": "0.3.20",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz",
"integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==",
"dev": true,
"dependencies": {
"@jridgewell/resolve-uri": "^3.1.0",
@ -2038,14 +2038,14 @@
}
},
"node_modules/@sentry-internal/tracing": {
"version": "7.74.0",
"resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.0.tgz",
"integrity": "sha512-JK6IRGgdtZjswGfaGIHNWIThffhOHzVIIaGmglui+VFIzOsOqePjoxaDV0MEvzafxXZD7eWqGE5RGuZ0n6HFVg==",
"version": "7.74.1",
"resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.1.tgz",
"integrity": "sha512-nNaiZreQxCitG2PzYPaC7XtyA9OMsETGYMKAtiK4p62/uTmeYbsBva9BoNx1XeiHRwbrVQYRMKQ9nV5e2jS4/A==",
"dev": true,
"dependencies": {
"@sentry/core": "7.74.0",
"@sentry/types": "7.74.0",
"@sentry/utils": "7.74.0",
"@sentry/core": "7.74.1",
"@sentry/types": "7.74.1",
"@sentry/utils": "7.74.1",
"tslib": "^2.4.1 || ^1.9.3"
},
"engines": {
@ -2092,13 +2092,13 @@
}
},
"node_modules/@sentry/core": {
"version": "7.74.0",
"resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.0.tgz",
"integrity": "sha512-83NRuqn7nDZkSVBN5yJQqcpXDG4yMYiB7TkYUKrGTzBpRy6KUOrkCdybuKk0oraTIGiGSe5WEwCFySiNgR9FzA==",
"version": "7.74.1",
"resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.1.tgz",
"integrity": "sha512-LvEhOSfdIvwkr+PdlrT/aA/iOLhkXrSkvjqAQyogE4ddCWeYfS0NoirxNt1EaxMBAWKhYZRqzkA7WA4LDLbzlA==",
"dev": true,
"dependencies": {
"@sentry/types": "7.74.0",
"@sentry/utils": "7.74.0",
"@sentry/types": "7.74.1",
"@sentry/utils": "7.74.1",
"tslib": "^2.4.1 || ^1.9.3"
},
"engines": {
@ -2172,15 +2172,15 @@
}
},
"node_modules/@sentry/node": {
"version": "7.74.0",
"resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.0.tgz",
"integrity": "sha512-uBmW2/z0cz/WFIG74ZF7lSipO0XNzMf9yrdqnZXnGDYsUZE4I4QiqDN0hNi6fkTgf9MYRC8uFem2OkAvyPJ74Q==",
"version": "7.74.1",
"resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.1.tgz",
"integrity": "sha512-aMUQ2LFZF64FBr+cgjAqjT4OkpYBIC9lyWI8QqjEHqNho5+LGu18/iVrJPD4fgs4UhGdCuAiQjpC36MbmnIDZA==",
"dev": true,
"dependencies": {
"@sentry-internal/tracing": "7.74.0",
"@sentry/core": "7.74.0",
"@sentry/types": "7.74.0",
"@sentry/utils": "7.74.0",
"@sentry-internal/tracing": "7.74.1",
"@sentry/core": "7.74.1",
"@sentry/types": "7.74.1",
"@sentry/utils": "7.74.1",
"cookie": "^0.5.0",
"https-proxy-agent": "^5.0.0",
"lru_map": "^0.3.3",
@ -2191,21 +2191,21 @@
}
},
"node_modules/@sentry/types": {
"version": "7.74.0",
"resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.0.tgz",
"integrity": "sha512-rI5eIRbUycWjn6s6o3yAjjWtIvYSxZDdnKv5je2EZINfLKcMPj1dkl6wQd2F4y7gLfD/N6Y0wZYIXC3DUdJQQg==",
"version": "7.74.1",
"resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.1.tgz",
"integrity": "sha512-2jIuPc+YKvXqZETwr2E8VYnsH1zsSUR/wkIvg1uTVeVNyoowJv+YsOtCdeGyL2AwiotUBSPKu7O1Lz0kq5rMOQ==",
"dev": true,
"engines": {
"node": ">=8"
}
},
"node_modules/@sentry/utils": {
"version": "7.74.0",
"resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.0.tgz",
"integrity": "sha512-k3np8nuTPtx5KDODPtULfFln4UXdE56MZCcF19Jv6Ljxf+YN/Ady1+0Oi3e0XoSvFpWNyWnglauT7M65qCE6kg==",
"version": "7.74.1",
"resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.1.tgz",
"integrity": "sha512-qUsqufuHYcy5gFhLZslLxA5kcEOkkODITXW3c7D+x+8iP/AJqa8v8CeUCVNS7RetHCuIeWAbbTClC4c411EwQg==",
"dev": true,
"dependencies": {
"@sentry/types": "7.74.0",
"@sentry/types": "7.74.1",
"tslib": "^2.4.1 || ^1.9.3"
},
"engines": {
@ -2237,9 +2237,9 @@
}
},
"node_modules/@types/babel__core": {
"version": "7.20.2",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.2.tgz",
"integrity": "sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA==",
"version": "7.20.3",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.3.tgz",
"integrity": "sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA==",
"dev": true,
"dependencies": {
"@babel/parser": "^7.20.7",
@ -2250,18 +2250,18 @@
}
},
"node_modules/@types/babel__generator": {
"version": "7.6.5",
"resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.5.tgz",
"integrity": "sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w==",
"version": "7.6.6",
"resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.6.tgz",
"integrity": "sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w==",
"dev": true,
"dependencies": {
"@babel/types": "^7.0.0"
}
},
"node_modules/@types/babel__template": {
"version": "7.4.2",
"resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.2.tgz",
"integrity": "sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ==",
"version": "7.4.3",
"resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.3.tgz",
"integrity": "sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ==",
"dev": true,
"dependencies": {
"@babel/parser": "^7.1.0",
@ -2269,18 +2269,18 @@
}
},
"node_modules/@types/babel__traverse": {
"version": "7.20.2",
"resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.2.tgz",
"integrity": "sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw==",
"version": "7.20.3",
"resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.3.tgz",
"integrity": "sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw==",
"dev": true,
"dependencies": {
"@babel/types": "^7.20.7"
}
},
"node_modules/@types/better-sqlite3": {
"version": "7.6.5",
"resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.5.tgz",
"integrity": "sha512-H3ZUx89KiPhYa9nalUXVVStSUFHuzYxt4yoazufpTTYW9rVUCzhh02V8CH2C8nE4libnK0UgFq5DFIe0DOhqow==",
"version": "7.6.6",
"resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.6.tgz",
"integrity": "sha512-nuFAptzt0hZYBvyLzKQCbuCCK+RN9PHH4ezar5EJLIg2qpVhwQ/uLvLO/K8A9O7N8DafawgFupiyXQSs0U48Ng==",
"dev": true,
"dependencies": {
"@types/node": "*"
@ -3536,9 +3536,9 @@
}
},
"node_modules/electron-to-chromium": {
"version": "1.4.554",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.554.tgz",
"integrity": "sha512-Q0umzPJjfBrrj8unkONTgbKQXzXRrH7sVV7D9ea2yBV3Oaogz991yhbpfvo2LMNkJItmruXTEzVpP9cp7vaIiQ==",
"version": "1.4.557",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.557.tgz",
"integrity": "sha512-6x0zsxyMXpnMJnHrondrD3SuAeKcwij9S+83j2qHAQPXbGTDDfgImzzwgGlzrIcXbHQ42tkG4qA6U860cImNhw==",
"dev": true
},
"node_modules/emittery": {

View file

@ -61,6 +61,7 @@ const populateTweetProperties = async (
name: apiUser.name,
screen_name: apiUser.screen_name,
avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '',
// @ts-expect-error Legacy api shit
avatar_color: null,
banner_url: apiUser.banner_url || '',
description: apiUser.description || '',
@ -77,8 +78,11 @@ const populateTweetProperties = async (
};
apiTweet.replies = tweet.legacy.reply_count;
apiTweet.retweets = tweet.legacy.retweet_count;
apiTweet.reposts = tweet.legacy.retweet_count;
apiTweet.likes = tweet.legacy.favorite_count;
// @ts-expect-error Legacy api shit
apiTweet.color = null;
// @ts-expect-error legacy api
apiTweet.twitter_card = 'tweet';
apiTweet.created_at = tweet.legacy.created_at;
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
@ -131,11 +135,11 @@ const populateTweetProperties = async (
apiTweet.media.all.push(mediaObject);
if (mediaObject.type === 'photo') {
apiTweet.twitter_card = 'summary_large_image';
apiTweet.embed_card = 'summary_large_image';
apiTweet.media.photos = apiTweet.media.photos || [];
apiTweet.media.photos.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
apiTweet.twitter_card = 'player';
apiTweet.embed_card = 'player';
apiTweet.media.videos = apiTweet.media.videos || [];
apiTweet.media.videos.push(mediaObject);
} else {
@ -183,9 +187,9 @@ const populateTweetProperties = async (
/* Workaround: Force player card by default for videos */
/* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
The mediaList however can set it to something else. TODO: Reimplement as enums */
// @ts-expect-error see above comment
if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') {
apiTweet.twitter_card = 'player';
if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
apiTweet.embed_card = 'player';
}
/* If a language is specified in API or by user, let's try translating it! */

View file

@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
apiUser.following = user.legacy.friends_count;
apiUser.likes = user.legacy.favourites_count;
apiUser.tweets = user.legacy.statuses_count;
apiUser.posts = user.legacy.statuses_count;
apiUser.name = user.legacy.name;
apiUser.screen_name = user.legacy.screen_name;
apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description);
apiUser.location = user.legacy.location;
apiUser.banner_url = user.legacy.profile_banner_url;
apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`
apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null;
apiUser.location = user.legacy.location ? user.legacy.location : null;
apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null;
/*
if (user.is_blue_verified) {
apiUser.verified = 'blue';

View file

@ -15,6 +15,7 @@ export const Constants = {
RELEASE_NAME: RELEASE_NAME,
API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`,
TWITTER_ROOT: 'https://twitter.com',
TWITTER_GLOBAL_NAME_ROOT: 'twitter.com',
TWITTER_API_ROOT: 'https://api.twitter.com',
BOT_UA_REGEX:
/bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi,

View file

@ -120,7 +120,7 @@ export const handleStatus = async (
const headers = [
`<link rel="canonical" href="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
`<meta property="og:url" content="${Constants.TWITTER_ROOT}/${tweet.author.screen_name}/status/${tweet.id}"/>`,
`<meta property="theme-color" content="${tweet.color || '#00a8fc'}"/>`,
`<meta property="theme-color" content="#00a8fc"/>`,
`<meta property="twitter:site" content="@${tweet.author.screen_name}"/>`,
`<meta property="twitter:creator" content="@${tweet.author.screen_name}"/>`,
`<meta property="twitter:title" content="${tweet.author.name} (@${tweet.author.screen_name})"/>`
@ -199,8 +199,8 @@ export const handleStatus = async (
siteName = instructions.siteName;
}
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
if (tweet.twitter_card === 'player') {
tweet.twitter_card = 'summary_large_image';
if (tweet.embed_card === 'player') {
tweet.embed_card = 'summary_large_image';
}
break;
case 'video':
@ -216,8 +216,8 @@ export const handleStatus = async (
siteName = instructions.siteName;
}
/* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */
if (tweet.twitter_card !== 'player') {
tweet.twitter_card = 'player';
if (tweet.embed_card !== 'player') {
tweet.embed_card = 'player';
}
/* This Tweet has a video to render. */
break;
@ -344,7 +344,7 @@ export const handleStatus = async (
and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */
const text = useIV ? sanitizeText(newText).replace(/\n/g, '<br>') : sanitizeText(newText);
const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card;
const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card;
/* Push basic headers relating to author, Tweet text, and site name */
headers.push(

View file

@ -1,6 +1,6 @@
/* We keep this value up-to-date for making our requests to Twitter as
indistinguishable from normal user traffic as possible. */
const fakeChromeVersion = 116;
const fakeChromeVersion = 118;
const platformWindows = 'Windows NT 10.0; Win64; x64';
const platformMac = 'Macintosh; Intel Mac OS X 10_15_7';
const platformLinux = 'X11; Linux x86_64';

View file

@ -0,0 +1,226 @@
import { renderCard } from '../../helpers/card';
import { Constants } from '../../constants';
import { linkFixer } from '../../helpers/linkFixer';
import { handleMosaic } from '../../helpers/mosaic';
// import { translateTweet } from '../../helpers/translate';
import { unescapeText } from '../../helpers/utils';
import { processMedia } from '../../helpers/media';
import { convertToApiUser } from '../../api/user';
export const buildAPITweet = async (
tweet: GraphQLTweet,
language: string | undefined,
threadPiece = false,
legacyAPI = false
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<APITweet | null> => {
const apiTweet = {} as APITweet;
/* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */
if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') {
tweet = tweet.result;
}
if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') {
tweet.core = tweet.tweet.core;
}
if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') {
tweet.legacy = tweet.tweet?.legacy;
}
if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') {
tweet.views = tweet?.tweet?.views;
}
if (typeof tweet.core === 'undefined') {
console.log('Tweet still not valid', tweet);
return null;
}
const graphQLUser = tweet.core.user_results.result;
const apiUser = convertToApiUser(graphQLUser);
/* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */
const id = tweet.rest_id ?? tweet.legacy.id_str;
/* Populating a lot of the basics */
apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
apiTweet.id = id;
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || ''));
if (!threadPiece) {
apiTweet.author = {
id: apiUser.id,
name: apiUser.name,
screen_name: apiUser.screen_name,
global_screen_name: apiUser.global_screen_name,
avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null,
banner_url: apiUser.banner_url ?? null,
description: apiUser.description ?? null,
location: apiUser.location ?? null,
url: apiUser.url ?? null,
followers: apiUser.followers,
following: apiUser.following,
joined: apiUser.joined,
posts: apiUser.tweets,
likes: apiUser.likes,
protected: apiUser.protected,
birthday: apiUser.birthday,
website: apiUser.website
};
}
apiTweet.replies = tweet.legacy.reply_count;
if (legacyAPI) {
apiTweet.retweets = tweet.legacy.retweet_count;
} else {
apiTweet.reposts = tweet.legacy.retweet_count;
}
apiTweet.likes = tweet.legacy.favorite_count;
apiTweet.embed_card = 'tweet';
apiTweet.created_at = tweet.legacy.created_at;
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive;
if (tweet.views.state === 'EnabledWithCount') {
apiTweet.views = parseInt(tweet.views.count || '0') ?? null;
} else {
apiTweet.views = null;
}
console.log('note_tweet', JSON.stringify(tweet.note_tweet));
const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text;
if (noteTweetText) {
tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls;
tweet.legacy.entities.hashtags =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags;
tweet.legacy.entities.symbols =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols;
console.log('We meet the conditions to use new note tweets');
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText));
apiTweet.is_note_tweet = true;
} else {
apiTweet.is_note_tweet = false;
}
if (tweet.legacy.lang !== 'unk') {
apiTweet.lang = tweet.legacy.lang;
} else {
apiTweet.lang = null;
}
if (legacyAPI) {
apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
} else if (tweet.legacy.in_reply_to_screen_name) {
apiTweet.reply_of = {
screen_name: tweet.legacy.in_reply_to_screen_name || null,
post: tweet.legacy.in_reply_to_status_id_str || null
};
} else {
apiTweet.reply_of = null;
}
apiTweet.media = {
all: [],
photos: [],
videos: [],
};
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet;
/* Only override the embed_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.embed_card === 'tweet'&& apiTweet.quote !== null) {
apiTweet.embed_card = apiTweet.quote.embed_card;
}
}
const mediaList = Array.from(
tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []
);
// console.log('tweet', JSON.stringify(tweet));
/* Populate this Tweet's media */
mediaList.forEach(media => {
const mediaObject = processMedia(media);
if (mediaObject) {
apiTweet.media?.all?.push(mediaObject);
if (mediaObject.type === 'photo') {
apiTweet.embed_card = 'summary_large_image';
apiTweet.media?.photos?.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
apiTweet.embed_card = 'player';
apiTweet.media?.videos?.push(mediaObject);
} else {
console.log('Unknown media type', mediaObject.type);
}
}
});
/* Grab color palette data */
/*
if (mediaList[0]?.ext_media_color?.palette) {
apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette);
}
*/
/* Handle photos and mosaic if available */
if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) {
const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
apiTweet.media.mosaic = mosaic;
}
}
// Add Tweet source but remove the link HTML tag
if (tweet.source) {
apiTweet.source = (tweet.source || '').replace(
/<a href="(.+?)" rel="nofollow">(.+?)<\/a>/,
'$2'
);
}
/* Populate a Twitter card */
if (tweet.card) {
const card = renderCard(tweet.card);
if (card.external_media) {
apiTweet.media = apiTweet.media ?? {};
apiTweet.media.external = card.external_media;
}
if (card.poll) {
apiTweet.poll = card.poll;
}
}
/* Workaround: Force player card by default for videos */
/* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
The mediaList however can set it to something else. TODO: Reimplement as enums */
// @ts-expect-error see above comment
if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
apiTweet.embed_card = 'player';
}
/* If a language is specified in API or by user, let's try translating it! */
if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
/* TODO: Reimplement */
// console.log(`Attempting to translate Tweet to ${language}...`);
// const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language);
// if (translateAPI !== null && translateAPI?.translation) {
// apiTweet.translation = {
// text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')),
// source_lang: translateAPI?.sourceLanguage || '',
// target_lang: translateAPI?.destinationLanguage || '',
// source_lang_en: translateAPI?.localizedSourceLanguage || ''
// };
// }
}
return apiTweet;
};

View file

@ -0,0 +1,308 @@
import { IRequest } from "itty-router";
import { Constants } from "../../constants";
import { twitterFetch } from "../../fetch";
import { buildAPITweet } from "./processor";
type GraphQLProcessBucket = {
tweets: GraphQLTweet[];
cursors: GraphQLTimelineCursor[];
}
type SocialThread = {
post: APIPost | APITweet | null;
thread: (APIPost | APITweet)[] | null;
author: APIUser | null;
}
export const fetchTwitterThread = async (
status: string,
event: FetchEvent,
useElongator = typeof TwitterProxy !== 'undefined',
cursor: string | null = null
): Promise<GraphQLTweetFoundResponse> => {
return (await twitterFetch(
`${
Constants.TWITTER_ROOT
}/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent(
JSON.stringify({
focalTweetId: status,
referrer: "home",
with_rux_injections: false,
includePromotedContent: false,
withCommunity: true,
withBirdwatchNotes: true,
withQuickPromoteEligibilityTweetFields: false,
withVoice: false,
withV2Timeline: true,
cursor: cursor
})
)}&features=${encodeURIComponent(
JSON.stringify({
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
responsive_web_home_pinned_timelines_enabled: true,
creator_subscriptions_tweet_preview_api_enabled: true,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
longform_notetweets_rich_text_read_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_media_download_video_enabled: true,
responsive_web_enhance_cards_enabled: true
})
)}&fieldToggles=${encodeURIComponent(
JSON.stringify({
withArticleRichContentState: true
})
)}`,
event,
useElongator,
() => {
return true;
}
)) as GraphQLTweetFoundResponse;
};
const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => {
const bucket: GraphQLProcessBucket = {
tweets: [],
cursors: []
};
instructions.forEach?.(instruction => {
if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') {
// @ts-expect-error Use entries or moduleItems depending on the type
(instruction?.entries ?? instruction.moduleItems).forEach((_entry) => {
const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry
const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content;
if (content.__typename === 'TimelineTimelineItem') {
const itemContentType = content.itemContent?.__typename;
if (itemContentType === 'TimelineTweet') {
const entryType = content.itemContent.tweet_results.result.__typename
if (entryType === 'Tweet') {
bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet);
}
if (entryType === 'TweetWithVisibilityResults') {
bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet);
}
} else if (itemContentType === 'TimelineTimelineCursor') {
bucket.cursors.push(content.itemContent as GraphQLTimelineCursor);
}
} else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') {
content.items.forEach((item) => {
const itemContentType = item.item.itemContent.__typename;
if (itemContentType === 'TimelineTweet') {
const entryType = item.item.itemContent.tweet_results.result.__typename
if (entryType === 'Tweet') {
bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet);
}
if (entryType === 'TweetWithVisibilityResults') {
bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet);
}
} else if (itemContentType === 'TimelineTimelineCursor') {
bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor);
}
});
}
});
}
})
return bucket;
}
const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => {
return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null;
}
const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => {
return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id);
}
const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => {
const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id);
if (!tweet) {
console.log('uhhh, we could not even find that tweet, dunno how that happened');
return -1;
}
const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str);
if (index === -1) {
console.log('could not find shit for', id)
console.log(bucket.cursors)
}
return index;
}
const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => {
/* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */
return oldCursors.map(cursor => {
const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType);
if (newCursor) {
return newCursor;
}
return cursor;
});
}
const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id)
}
export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise<SocialThread> => {
const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse;
if (!response.data) {
return { post: null, thread: null, author: null };
}
const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions);
const originalTweet = findTweetInBucket(id, bucket);
/* Don't bother processing thread on a null tweet */
if (originalTweet === null) {
return { post: null, thread: null, author: null };
}
const post = await buildAPITweet(originalTweet, undefined, false, false);
const author = post.author;
/* remove post.author */
// @ts-expect-error lmao
delete post.author;
/* If we're not processing threads, let's be done here */
if (!processThread) {
return { post: post, thread: null, author: author };
}
const threadTweets = [originalTweet];
bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet);
let currentId = id;
/* Process tweets that are following the current one in the thread */
while (findNextTweet(currentId, bucket) !== -1) {
const index = findNextTweet(currentId, bucket);
const tweet = bucket.tweets[index];
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
threadTweets.push(tweet);
currentId = newCurrentId;
console.log('Current index', index, 'of', bucket.tweets.length)
/* Reached the end of the current list of tweets in thread) */
if (index >= (bucket.tweets.length - 1)) {
/* See if we have a cursor to fetch more tweets */
const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore'));
console.log('current cursors: ', bucket.cursors)
if (!cursor) {
console.log('No cursor present, stopping pagination down')
break;
}
console.log('Cursor present, fetching more tweets down');
let loadCursor: GraphQLTweetFoundResponse;
try {
loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
console.log('Unknown data while fetching cursor', loadCursor);
break;
}
} catch(e) {
console.log('Error fetching cursor', e);
break;
}
const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet));
/* Remove old cursor and add new bottom cursor if necessary */
consolidateCursors(bucket.cursors, cursorResponse.cursors);
console.log('updated bucket of cursors', bucket.cursors);
}
console.log('Preview of next tweet:', findNextTweet(currentId, bucket));
}
currentId = id;
while (findPreviousTweet(currentId, bucket) !== -1) {
const index = findPreviousTweet(currentId, bucket);
const tweet = bucket.tweets[index];
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
threadTweets.unshift(tweet);
currentId = newCurrentId;
if (index === 0) {
/* See if we have a cursor to fetch more tweets */
const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore'));
console.log('current cursors: ', bucket.cursors)
if (!cursor) {
console.log('No cursor present, stopping pagination up')
break;
}
console.log('Cursor present, fetching more tweets up');
let loadCursor: GraphQLTweetFoundResponse;
try {
loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value)
if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
console.log('Unknown data while fetching cursor', loadCursor);
break;
}
} catch(e) {
console.log('Error fetching cursor', e);
break;
}
const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet));
/* Remove old cursor and add new top cursor if necessary */
consolidateCursors(bucket.cursors, cursorResponse.cursors);
// console.log('updated bucket of tweets', bucket.tweets);
console.log('updated bucket of cursors', bucket.cursors);
}
console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket));
}
const socialThread: SocialThread = {
post: post,
thread: [],
author: author
}
threadTweets.forEach(async (tweet) => {
socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false));
});
return socialThread;
}
export const threadAPIProvider = async (request: IRequest) => {
const { id } = request.params;
const processedResponse = await processTwitterThread(id, true, request);
return new Response(JSON.stringify(processedResponse), {
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }
})
}

View file

@ -354,7 +354,7 @@ type GraphQLTweetLegacy = {
type GraphQLTweet = {
// Workaround
result: GraphQLTweet;
__typename: 'Tweet' | 'TweetUnavailable';
__typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable';
rest_id: string; // "1674824189176590336",
has_birdwatch_notes: false;
core: {
@ -444,37 +444,76 @@ type TweetTombstone = {
};
};
};
type GraphQLTimelineTweet = {
item: 'TimelineTweet';
__typename: 'TimelineTweet';
tweet_results: {
result: GraphQLTweet | TweetTombstone;
};
}
type GraphQLTimelineCursor = {
cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore';
itemType: 'TimelineTimelineCursor';
value: string;
__typename: 'TimelineTimelineCursor';
}
interface GraphQLBaseTimeline {
entryType: string;
__typename: string;
}
type GraphQLTimelineItem = GraphQLBaseTimeline & {
entryType: 'TimelineTimelineItem';
__typename: 'TimelineTimelineItem';
itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor;
}
type GraphQLTimelineModule = GraphQLBaseTimeline & {
entryType: 'TimelineTimelineModule';
__typename: 'TimelineTimelineModule';
items: {
entryId: `conversationthread-${number}-tweet-${number}`;
item: GraphQLTimelineItem
}[];
}
type GraphQLTimelineTweetEntry = {
/** The entryID contains the tweet ID */
entryId: `tweet-${number}`; // "tweet-1674824189176590336"
sortIndex: string;
content: {
entryType: 'TimelineTimelineItem';
__typename: 'TimelineTimelineItem';
itemContent: {
item: 'TimelineTweet';
__typename: 'TimelineTweet';
tweet_results: {
result: GraphQLTweet | TweetTombstone;
};
};
};
content: GraphQLTimelineItem;
};
type GraphQLModuleTweetEntry = {
/** The entryID contains the tweet ID */
sortIndex: string;
item: GraphQLTimelineItem | GraphQLTimelineModule;
};
type GraphQLConversationThread = {
entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336"
sortIndex: string;
content: GraphQLTimelineModule;
};
type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown;
type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction;
type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction;
type TimeLineAddEntriesInstruction = {
type TimelineAddEntriesInstruction = {
type: 'TimelineAddEntries';
entries: GraphQLTimelineEntry[];
};
type TimeLineTerminateTimelineInstruction = {
type TimelineAddModulesInstruction = {
type: 'TimelineAddToModule';
moduleItems: GraphQLTimelineEntry[];
};
type TimelineTerminateTimelineInstruction = {
type: 'TimelineTerminateTimeline';
direction: 'Top';
};

51
src/types/types.d.ts vendored
View file

@ -81,14 +81,6 @@ interface APITranslate {
target_lang: string;
}
interface BaseUser {
id?: string;
name?: string;
screen_name?: string;
avatar_url?: string;
banner_url?: string;
}
interface APIExternalMedia {
type: 'video';
url: string;
@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia {
};
}
interface APITweet {
interface APIPost {
id: string;
url: string;
text: string;
@ -144,15 +136,11 @@ interface APITweet {
created_timestamp: number;
likes: number;
retweets: number;
reposts: number;
replies: number;
views?: number | null;
color: string | null;
quote?: APITweet;
quote?: APIPost;
poll?: APIPoll;
translation?: APITranslate;
author: APIUser;
media?: {
@ -169,24 +157,41 @@ interface APITweet {
replying_to: string | null;
replying_to_status: string | null;
source: string;
reply_of: {
screen_name: string | null;
post: string | null;
} | null
is_note_tweet: boolean;
source: string | null;
twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player';
}
interface APIUser extends BaseUser {
interface APITweet extends APIPost {
retweets: number;
views?: number | null;
translation?: APITranslate;
is_note_tweet: boolean;
}
interface APIUser {
id: string;
name: string;
screen_name: string;
global_screen_name: string;
avatar_url: string | null;
banner_url: string | null;
// verified: 'legacy' | 'blue'| 'business' | 'government';
// verified_label: string;
description: string;
location: string;
description: string | null;
location: string | null;
url: string;
avatar_color?: string | null;
protected: boolean;
followers: number;
following: number;
tweets: number;
tweets?: number;
posts?: number;
likes: number;
joined: string;
website: {

View file

@ -10,6 +10,7 @@ import { Strings } from './strings';
import motd from '../motd.json';
import { sanitizeText } from './helpers/utils';
import { handleProfile } from './user';
import { threadAPIProvider } from './providers/twitter/status';
declare const globalThis: {
fetchCompletedTime: number;
@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest);
router.get('/status/:id/:language', statusRequest);
router.get('/version', versionRequest);
router.get('/set_base_redirect', setRedirectRequest);
router.get('/v2/twitter/thread/:id', threadAPIProvider)
/* Oembeds (used by Discord to enhance responses)