Use proto-v2 thread API internally

This commit is contained in:
dangered wolf 2023-11-02 03:26:19 -04:00
parent ae7c432481
commit eb90f48226
No known key found for this signature in database
GPG key ID: 41E4D37680ED8B58
9 changed files with 67 additions and 359 deletions

View file

@ -1,315 +0,0 @@
import { renderCard } from '../helpers/card';
import { Constants } from '../constants';
import { fetchConversation } from '../fetch';
import { linkFixer } from '../helpers/linkFixer';
import { handleMosaic } from '../helpers/mosaic';
import { translateTweet } from '../helpers/translate';
import { unescapeText } from '../helpers/utils';
import { processMedia } from '../helpers/media';
import { convertToApiUser } from './user';
import { isGraphQLTweet } from '../helpers/graphql';
/* This function does the heavy lifting of processing data from Twitter API
and using it to create FixTweet's streamlined API responses */
const populateTweetProperties = async (
tweet: GraphQLTweet,
conversation: TweetResultsByRestIdResult, // TimelineBlobPartial,
language: string | undefined
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<APITweet | null> => {
const apiTweet = {} as APITweet;
/* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */
if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') {
tweet = tweet.result;
}
if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') {
tweet.core = tweet.tweet.core;
}
if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') {
tweet.legacy = tweet.tweet?.legacy;
}
if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') {
tweet.views = tweet?.tweet?.views;
}
if (typeof tweet.core === 'undefined') {
console.log('Tweet still not valid', tweet);
return null;
}
/* With v2 conversation API we re-add the user object ot the tweet because
Twitter stores it separately in the conversation API. This is to consolidate
it in case a user appears multiple times in a thread. */
const graphQLUser = tweet.core.user_results.result;
const apiUser = convertToApiUser(graphQLUser);
/* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */
const id = tweet.rest_id ?? tweet.legacy.id_str;
/* Populating a lot of the basics */
apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
apiTweet.id = id;
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || ''));
apiTweet.author = {
id: apiUser.id,
name: apiUser.name,
screen_name: apiUser.screen_name,
avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '',
// @ts-expect-error Legacy api shit
avatar_color: null,
banner_url: apiUser.banner_url || '',
description: apiUser.description || '',
location: apiUser.location || '',
url: apiUser.url || '',
followers: apiUser.followers,
following: apiUser.following,
joined: apiUser.joined,
tweets: apiUser.tweets,
likes: apiUser.likes,
protected: apiUser.protected,
birthday: apiUser.birthday,
website: apiUser.website
};
apiTweet.replies = tweet.legacy.reply_count;
apiTweet.reposts = tweet.legacy.retweet_count;
apiTweet.reposts = tweet.legacy.retweet_count;
apiTweet.likes = tweet.legacy.favorite_count;
// @ts-expect-error Legacy api shit
apiTweet.color = null;
// @ts-expect-error legacy api
apiTweet.twitter_card = 'tweet';
apiTweet.created_at = tweet.legacy.created_at;
apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000;
apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive;
if (tweet.views.state === 'EnabledWithCount') {
apiTweet.views = parseInt(tweet.views.count || '0') ?? null;
} else {
apiTweet.views = null;
}
console.log('note_tweet', JSON.stringify(tweet.note_tweet));
const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text;
if (noteTweetText) {
tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls;
tweet.legacy.entities.hashtags =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags;
tweet.legacy.entities.symbols =
tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols;
console.log('We meet the conditions to use new note tweets');
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText));
apiTweet.is_note_tweet = true;
} else {
apiTweet.is_note_tweet = false;
}
if (tweet.legacy.lang !== 'unk') {
apiTweet.lang = tweet.legacy.lang;
} else {
apiTweet.lang = null;
}
apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
const mediaList = Array.from(
tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []
);
// console.log('tweet', JSON.stringify(tweet));
/* Populate this Tweet's media */
mediaList.forEach(media => {
const mediaObject = processMedia(media);
if (mediaObject) {
apiTweet.media = apiTweet.media || {};
apiTweet.media.all = apiTweet.media?.all || [];
apiTweet.media.all.push(mediaObject);
if (mediaObject.type === 'photo') {
apiTweet.embed_card = 'summary_large_image';
apiTweet.media.photos = apiTweet.media.photos || [];
apiTweet.media.photos.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
apiTweet.embed_card = 'player';
apiTweet.media.videos = apiTweet.media.videos || [];
apiTweet.media.videos.push(mediaObject);
} else {
console.log('Unknown media type', mediaObject.type);
}
}
});
/* Grab color palette data */
/*
if (mediaList[0]?.ext_media_color?.palette) {
apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette);
}
*/
/* Handle photos and mosaic if available */
if ((apiTweet?.media?.photos?.length || 0) > 1) {
const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
apiTweet.media.mosaic = mosaic;
}
}
// Add Tweet source but remove the link HTML tag
if (tweet.source) {
apiTweet.source = (tweet.source || '').replace(
/<a href="(.+?)" rel="nofollow">(.+?)<\/a>/,
'$2'
);
}
/* Populate a Twitter card */
if (tweet.card) {
const card = renderCard(tweet.card);
if (card.external_media) {
apiTweet.media = apiTweet.media || {};
apiTweet.media.external = card.external_media;
}
if (card.poll) {
apiTweet.poll = card.poll;
}
}
/* Workaround: Force player card by default for videos */
/* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player'
The mediaList however can set it to something else. TODO: Reimplement as enums */
if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
apiTweet.embed_card = 'player';
}
/* If a language is specified in API or by user, let's try translating it! */
if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) {
console.log(`Attempting to translate Tweet to ${language}...`);
const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language);
if (translateAPI !== null && translateAPI?.translation) {
apiTweet.translation = {
text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')),
source_lang: translateAPI?.sourceLanguage || '',
target_lang: translateAPI?.destinationLanguage || '',
source_lang_en: translateAPI?.localizedSourceLanguage || ''
};
}
}
return apiTweet;
};
const writeDataPoint = (
event: FetchEvent,
language: string | undefined,
nsfw: boolean,
returnCode: string,
flags?: InputFlags
) => {
console.log('Writing data point...');
if (typeof AnalyticsEngine !== 'undefined') {
const flagString =
Object.keys(flags || {})
// @ts-expect-error - TypeScript doesn't like iterating over the keys, but that's OK
.filter(flag => flags?.[flag])[0] || 'standard';
AnalyticsEngine.writeDataPoint({
blobs: [
event.request.cf?.colo as string /* Datacenter location */,
event.request.cf?.country as string /* Country code */,
event.request.headers.get('user-agent') ??
'' /* User agent (for aggregating bots calling) */,
returnCode /* Return code */,
flagString /* Type of request */,
language ?? '' /* For translate feature */
],
doubles: [nsfw ? 1 : 0 /* NSFW media = 1, No NSFW Media = 0 */],
indexes: [event.request.headers.get('cf-ray') ?? '' /* CF Ray */]
});
}
};
/* API for Twitter statuses (Tweets)
Used internally by FixTweet's embed service, or
available for free using api.fxtwitter.com. */
export const statusAPI = async (
status: string,
language: string | undefined,
event: FetchEvent,
flags?: InputFlags
): Promise<TweetAPIResponse> => {
const res = await fetchConversation(status, event);
const tweet = res.data?.tweetResult?.result;
if (!tweet) {
return { code: 404, message: 'NOT_FOUND' };
}
/* We're handling this in the actual fetch code now */
// if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
// wasMediaBlockedNSFW = true;
// res = await fetchConversation(status, event, true);
// }
// console.log(JSON.stringify(tweet))
if (tweet.__typename === 'TweetUnavailable') {
if ((tweet as { reason: string })?.reason === 'Protected') {
writeDataPoint(event, language, false, 'PRIVATE_TWEET', flags);
return { code: 401, message: 'PRIVATE_TWEET' };
// } else if (tweet.reason === 'NsfwLoggedOut') {
// // API failure as elongator should have handled this
// writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags);
// return { code: 500, message: 'API_FAIL' };
} else {
// Api failure at parsing status
writeDataPoint(event, language, false, 'API_FAIL', flags);
return { code: 500, message: 'API_FAIL' };
}
}
// If the tweet is not a graphQL tweet something went wrong
if (!isGraphQLTweet(tweet)) {
console.log('Tweet was not a valid tweet', tweet);
writeDataPoint(event, language, false, 'API_FAIL', flags);
return { code: 500, message: 'API_FAIL' };
}
/*
if (tweet.retweeted_status_id_str) {
tweet = conversation?.globalObjects?.tweets?.[tweet.retweeted_status_id_str] || {};
}
*/
if (!tweet) {
return { code: 404, message: 'NOT_FOUND' };
}
/* Creating the response objects */
const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse;
const apiTweet: APITweet = (await populateTweetProperties(tweet, res, language)) as APITweet;
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet;
/* Only override the twitter_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) {
apiTweet.embed_card = apiTweet.quote.embed_card;
}
}
/* Finally, staple the Tweet to the response and return it */
response.tweet = apiTweet;
writeDataPoint(event, language, false, 'OK', flags);
return response;
};

View file

@ -10,14 +10,14 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
apiUser.followers = user.legacy.followers_count;
apiUser.following = user.legacy.friends_count;
apiUser.likes = user.legacy.favourites_count;
// @ts-expect-error `tweets` is only part of legacy API
apiUser.tweets = user.legacy.statuses_count;
apiUser.posts = user.legacy.statuses_count;
apiUser.name = user.legacy.name;
apiUser.screen_name = user.legacy.screen_name;
apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`
apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null;
apiUser.location = user.legacy.location ? user.legacy.location : null;
apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null;
apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : '';
apiUser.location = user.legacy.location ? user.legacy.location : '';
apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : '';
/*
if (user.is_blue_verified) {
apiUser.verified = 'blue';

View file

@ -41,7 +41,7 @@ export const handleStatus = async (
fetchWithThreads = true;
}
const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined);
const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined, flags?.api ?? false);
const tweet = thread?.post as APITweet;
@ -395,7 +395,7 @@ export const handleStatus = async (
authorText = `↪ Replying to @${tweet.replying_to}`;
/* We'll assume it's a thread if it's a reply to themselves */
} else if (
tweet.replying_to === tweet.author.screen_name &&
tweet.replying_to?.screen_name === tweet.author.screen_name &&
authorText === Strings.DEFAULT_AUTHOR_TEXT
) {
authorText = `↪ A part of @${tweet.author.screen_name}'s thread`;

View file

@ -14,7 +14,7 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = {
[Experiment.ELONGATOR_BY_DEFAULT]: {
name: 'Elongator by default',
description: 'Enable Elongator by default (guest token lockout bypass)',
percentage: 0.6
percentage: 1
},
[Experiment.ELONGATOR_PROFILE_API]: {
name: 'Elongator profile API',

View file

@ -237,6 +237,8 @@ export const constructTwitterThread = async (id: string,
language: string | undefined,
legacyAPI = false): Promise<SocialThread> => {
console.log('legacyAPI', legacyAPI)
let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult;
let post: APITweet;
@ -246,7 +248,7 @@ export const constructTwitterThread = async (id: string,
const result = response?.data?.tweetResult?.result as GraphQLTweet;
if (typeof result?.tweet === "undefined") {
if (typeof result === "undefined") {
return { post: null, thread: null, author: null, code: 404 };
}

View file

@ -56,16 +56,15 @@ export const buildAPITweet = async (
id: apiUser.id,
name: apiUser.name,
screen_name: apiUser.screen_name,
global_screen_name: apiUser.global_screen_name,
avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null,
banner_url: apiUser.banner_url ?? null,
description: apiUser.description ?? null,
location: apiUser.location ?? null,
url: apiUser.url ?? null,
banner_url: apiUser.banner_url,
description: apiUser.description,
location: apiUser.location,
url: apiUser.url,
followers: apiUser.followers,
following: apiUser.following,
joined: apiUser.joined,
posts: apiUser.tweets,
posts: apiUser.posts,
likes: apiUser.likes,
protected: apiUser.protected,
birthday: apiUser.birthday,
@ -76,8 +75,19 @@ export const buildAPITweet = async (
if (legacyAPI) {
// @ts-expect-error Use retweets for legacy API
apiTweet.retweets = tweet.legacy.retweet_count;
// @ts-expect-error `tweets` is only part of legacy API
apiTweet.author.tweets = apiTweet.author.posts;
// @ts-expect-error Part of legacy API that we no longer are able to track
apiTweet.author.avatar_color = null;
// @ts-expect-error Use retweets for legacy API
delete apiTweet.reposts;
// @ts-expect-error Use tweets and not posts for legacy API
delete apiTweet.author.posts;
delete apiTweet.author.global_screen_name;
} else {
apiTweet.reposts = tweet.legacy.retweet_count;
apiTweet.author.global_screen_name = apiUser.global_screen_name;
}
apiTweet.likes = tweet.legacy.favorite_count;
apiTweet.embed_card = 'tweet';
@ -115,27 +125,25 @@ export const buildAPITweet = async (
}
if (legacyAPI) {
// @ts-expect-error Use replying_to string for legacy API
apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null;
// @ts-expect-error Use replying_to_status string for legacy API
apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null;
} else if (tweet.legacy.in_reply_to_screen_name) {
apiTweet.reply_of = {
apiTweet.replying_to = {
screen_name: tweet.legacy.in_reply_to_screen_name || null,
post: tweet.legacy.in_reply_to_status_id_str || null
};
} else {
apiTweet.reply_of = null;
apiTweet.replying_to = null;
}
apiTweet.media = {
all: [],
photos: [],
videos: [],
};
apiTweet.media = {};
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet;
apiTweet.quote = (await buildAPITweet(quoteTweet, language, threadPiece, legacyAPI)) as APITweet;
/* Only override the embed_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) {
apiTweet.embed_card = apiTweet.quote.embed_card;
@ -152,13 +160,16 @@ export const buildAPITweet = async (
mediaList.forEach(media => {
const mediaObject = processMedia(media);
if (mediaObject) {
apiTweet.media.all = apiTweet.media?.all ?? [];
apiTweet.media?.all?.push(mediaObject);
if (mediaObject.type === 'photo') {
apiTweet.embed_card = 'summary_large_image';
apiTweet.media?.photos?.push(mediaObject);
apiTweet.media.photos = apiTweet.media?.photos ?? [];
apiTweet.media.photos?.push(mediaObject);
} else if (mediaObject.type === 'video' || mediaObject.type === 'gif') {
apiTweet.embed_card = 'player';
apiTweet.media?.videos?.push(mediaObject);
apiTweet.media.videos = apiTweet.media?.videos ?? [];
apiTweet.media.videos?.push(mediaObject);
} else {
console.log('Unknown media type', mediaObject.type);
}
@ -173,7 +184,7 @@ export const buildAPITweet = async (
*/
/* Handle photos and mosaic if available */
if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) {
if ((apiTweet?.media.photos?.length || 0) > 1 && !threadPiece) {
const mosaic = await handleMosaic(apiTweet.media?.photos || [], id);
if (typeof apiTweet.media !== 'undefined' && mosaic !== null) {
apiTweet.media.mosaic = mosaic;
@ -193,7 +204,6 @@ export const buildAPITweet = async (
if (tweet.card) {
const card = renderCard(tweet.card);
if (card.external_media) {
apiTweet.media = apiTweet.media ?? {};
apiTweet.media.external = card.external_media;
}
if (card.poll) {
@ -201,7 +211,7 @@ export const buildAPITweet = async (
}
}
if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') {
if (apiTweet.media?.videos && apiTweet.media?.videos.length > 0 && apiTweet.embed_card !== 'player') {
apiTweet.embed_card = 'player';
}
@ -219,5 +229,18 @@ export const buildAPITweet = async (
}
}
if (legacyAPI) {
// @ts-expect-error Use twitter_card for legacy API
apiTweet.twitter_card = apiTweet.embed_card;
// @ts-expect-error Part of legacy API that we no longer are able to track
apiTweet.color = null
// @ts-expect-error Use twitter_card for legacy API
delete apiTweet.embed_card;
if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) {
// @ts-expect-error media is not required in legacy API if empty
delete apiTweet.media;
}
}
return apiTweet;
};

View file

@ -16,7 +16,7 @@ const populateUserLinks = (tweet: APIPost, text: string): string => {
return text;
};
const generateTweetMedia = (tweet: APITweet): string => {
const generateTweetMedia = (tweet: APIPost): string => {
let media = '';
if (tweet.media?.all?.length) {
tweet.media.all.forEach(mediaItem => {
@ -131,7 +131,7 @@ const generateTweetFooter = (tweet: APIPost, isQuote = false): string => {
<!-- Embed profile picture, display name, and screen name in table -->
{aboutSection}
`.format({
socialText: getSocialTextIV(tweet) || '',
socialText: getSocialTextIV(tweet as APITweet) || '',
viewOriginal: !isQuote ? `<a href="${tweet.url}">View original post</a>` : notApplicableComment,
aboutSection: isQuote
? ''
@ -161,13 +161,13 @@ const generateTweetFooter = (tweet: APIPost, isQuote = false): string => {
});
};
const generateTweet = (tweet: APITweet, isQuote = false): string => {
const generateTweet = (tweet: APIPost, isQuote = false): string => {
let text = paragraphify(sanitizeText(tweet.text), isQuote);
text = htmlifyLinks(text);
text = htmlifyHashtags(text);
text = populateUserLinks(tweet, text);
const translatedText = getTranslatedText(tweet, isQuote);
const translatedText = getTranslatedText(tweet as APITweet, isQuote);
return `<!-- Telegram Instant View -->
{quoteHeader}

20
src/types/types.d.ts vendored
View file

@ -130,7 +130,7 @@ interface APIPost {
poll?: APIPoll;
author: APIUser;
media?: {
media: {
external?: APIExternalMedia;
photos?: APIPhoto[];
videos?: APIVideo[];
@ -141,10 +141,7 @@ interface APIPost {
lang: string | null;
possibly_sensitive: boolean;
replying_to: string | null;
replying_to_status: string | null;
reply_of: {
replying_to: {
screen_name: string | null;
post: string | null;
} | null
@ -165,19 +162,18 @@ interface APIUser {
id: string;
name: string;
screen_name: string;
global_screen_name: string;
avatar_url: string | null;
banner_url: string | null;
global_screen_name?: string;
avatar_url: string;
banner_url: string;
// verified: 'legacy' | 'blue'| 'business' | 'government';
// verified_label: string;
description: string | null;
location: string | null;
description: string;
location: string;
url: string;
protected: boolean;
followers: number;
following: number;
tweets?: number;
posts?: number;
posts: number;
likes: number;
joined: string;
website: {

View file

@ -120,8 +120,10 @@ test('API fetch basic Tweet', async () => {
expect(tweet.author.avatar_url).toBeTruthy();
expect(tweet.author.banner_url).toBeTruthy();
expect(tweet.replies).toBeGreaterThan(0);
// @ts-expect-error retweets only in legacy API
expect(tweet.retweets).toBeGreaterThan(0);
expect(tweet.likes).toBeGreaterThan(0);
// @ts-expect-error twitter_card only in legacy API
expect(tweet.twitter_card).toEqual('tweet');
expect(tweet.created_at).toEqual('Tue Mar 21 20:50:14 +0000 2006');
expect(tweet.created_timestamp).toEqual(1142974214);