DevComp/fxtwitter-docker
1
0
Fork 0
mirror of https://github.com/CompeyDev/fxtwitter-docker.git synced 2025-04-03 17:40:56 +01:00
Code Issues Projects Releases Packages Wiki Activity

Prettier and additional fixes

Browse source
This commit is contained in:
dangered wolf 2023-11-02 04:52:14 -04:00
parent eb90f48226
commit fe61670e9f
No known key found for this signature in database
GPG key ID: 41E4D37680ED8B58
11 changed files with 269 additions and 212 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
esbuild.config.mjs
View file

@ -10,7 +10,10 @@ config();
const gitCommit = execSync('git rev-parse --short HEAD').toString().trim();
const gitCommitFull = execSync('git rev-parse HEAD').toString().trim();
const gitUrl = execSync('git remote get-url origin').toString().trim();
const gitBranch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim().replace(/[\\\/]/g, '-');
const gitBranch = execSync('git rev-parse --abbrev-ref HEAD')
.toString()
.trim()
.replace(/[\\\/]/g, '-');
let workerName = 'fixtweet';

6
src/api/user.ts
View file

@ -14,8 +14,10 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => {
apiUser.tweets = user.legacy.statuses_count;
apiUser.name = user.legacy.name;
apiUser.screen_name = user.legacy.screen_name;
apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`
apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : '';
apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`;
apiUser.description = user.legacy.description
? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description)
: '';
apiUser.location = user.legacy.location ? user.legacy.location : '';
apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : '';
/*

18
src/embed/status.ts
View file

@ -41,7 +41,13 @@ export const handleStatus = async (
fetchWithThreads = true;
}
const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined, flags?.api ?? false);
const thread = await constructTwitterThread(
status,
fetchWithThreads,
request,
undefined,
flags?.api ?? false
);
const tweet = thread?.post as APITweet;
@ -51,19 +57,19 @@ export const handleStatus = async (
tweet: tweet
};
switch(api.code) {
switch (api.code) {
case 200:
api.message = "OK";
api.message = 'OK';
break;
case 401:
api.message = "PRIVATE_TWEET";
api.message = 'PRIVATE_TWEET';
break;
case 404:
api.message = "NOT_FOUND";
api.message = 'NOT_FOUND';
break;
case 500:
console.log(api);
api.message = "API_FAIL";
api.message = 'API_FAIL';
break;
}

6
src/experiments.ts
View file

@ -1,7 +1,7 @@
export enum Experiment {
ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT',
ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API',
TWEET_DETAIL_API = 'TWEET_DETAIL_API',
TWEET_DETAIL_API = 'TWEET_DETAIL_API'
}
type ExperimentConfig = {
@ -24,8 +24,8 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = {
[Experiment.TWEET_DETAIL_API]: {
name: 'Tweet detail API',
description: 'Use Tweet Detail API (where available with elongator)',
percentage: 0.75
},
percentage: 0.5
}
};
export const experimentCheck = (experiment: Experiment, condition = true) => {

98
src/fetch.ts
View file

@ -1,7 +1,6 @@
import { Constants } from './constants';
import { Experiment, experimentCheck } from './experiments';
import { generateUserAgent } from './helpers/useragent';
import { isGraphQLTweet } from './helpers/graphql';
const API_ATTEMPTS = 3;
let wasElongatorDisabled = false;
@ -26,7 +25,8 @@ export const twitterFetch = async (
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
),
validateFunction: (response: unknown) => boolean
validateFunction: (response: unknown) => boolean,
elongatorRequired = false
): Promise<unknown> => {
let apiAttempts = 0;
let newTokenGenerated = false;
@ -163,6 +163,11 @@ export const twitterFetch = async (
/* We'll usually only hit this if we get an invalid response from Twitter.
It's uncommon, but it happens */
console.error('Unknown error while fetching from API', e);
/* Elongator returns strings to communicate downstream errors */
if (String(e).indexOf('Status not found')) {
console.log('Tweet was not found');
return {};
}
!useElongator &&
event &&
event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true }));
@ -181,7 +186,6 @@ export const twitterFetch = async (
!wasElongatorDisabled &&
!useElongator &&
typeof TwitterProxy !== 'undefined' &&
// @ts-expect-error This is safe due to optional chaining
(response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason ===
'NsfwLoggedOut'
) {
@ -201,6 +205,10 @@ export const twitterFetch = async (
if (!validateFunction(response)) {
console.log('Failed to fetch response, got', JSON.stringify(response));
if (elongatorRequired) {
console.log('Elongator was required, but we failed to fetch a valid response');
return {};
}
if (useElongator) {
console.log('Elongator request failed to validate, trying again without it');
wasElongatorDisabled = true;
@ -232,87 +240,6 @@ export const twitterFetch = async (
return {};
};
export const fetchConversation = async (
status: string,
event: FetchEvent,
useElongator = experimentCheck(
Experiment.ELONGATOR_BY_DEFAULT,
typeof TwitterProxy !== 'undefined'
)
): Promise<TweetResultsByRestIdResult> => {
return (await twitterFetch(
`${
Constants.TWITTER_ROOT
}/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent(
JSON.stringify({
tweetId: status,
withCommunity: false,
includePromotedContent: false,
withVoice: false
})
)}&features=${encodeURIComponent(
JSON.stringify({
creator_subscriptions_tweet_preview_api_enabled: true,
tweetypie_unmention_optimization_enabled: true,
responsive_web_edit_tweet_api_enabled: true,
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
view_counts_everywhere_api_enabled: true,
longform_notetweets_consumption_enabled: true,
responsive_web_twitter_article_tweet_consumption_enabled: false,
tweet_awards_web_tipping_enabled: false,
freedom_of_speech_not_reach_fetch_enabled: true,
standardized_nudges_misinfo: true,
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
longform_notetweets_rich_text_read_enabled: true,
longform_notetweets_inline_media_enabled: true,
responsive_web_graphql_exclude_directive_enabled: true,
verified_phone_label_enabled: false,
responsive_web_media_download_video_enabled: false,
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
responsive_web_graphql_timeline_navigation_enabled: true,
responsive_web_enhance_cards_enabled: false
})
)}&fieldToggles=${encodeURIComponent(
JSON.stringify({
withArticleRichContentState: true
})
)}`,
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as TweetResultsByRestIdResult;
// If we get a not found error it's still a valid response
const tweet = conversation.data?.tweetResult?.result;
if (isGraphQLTweet(tweet)) {
return true;
}
console.log('invalid graphql tweet');
if (
!tweet &&
typeof conversation.data?.tweetResult === 'object' &&
Object.keys(conversation.data?.tweetResult || {}).length === 0
) {
console.log('tweet was not found');
return true;
}
if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') {
console.log('tweet is nsfw');
return true;
}
if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
console.log('tweet is protected');
return true;
}
if (tweet?.__typename === 'TweetUnavailable') {
console.log('generic tweet unavailable error');
return true;
}
// Final clause for checking if it's valid is if there's errors
return Array.isArray(conversation.errors);
}
)) as TweetResultsByRestIdResult;
};
export const fetchUser = async (
username: string,
event: FetchEvent,
@ -359,6 +286,7 @@ export const fetchUser = async (
conversation.errors?.[0]?.code === 239)
);
*/
}
},
false
)) as GraphQLUserResponse;
};

2
src/helpers/mosaic.ts
View file

@ -13,7 +13,7 @@ const getDomain = (twitterId: string): string | null => {
hash = (hash << 5) - hash + char;
}
return mosaicDomains[Math.abs(hash) % mosaicDomains.length];
}
};
/* Handler for mosaic (multi-image combiner) */
export const handleMosaic = async (

276
src/providers/twitter/conversation.ts
View file

@ -1,23 +1,23 @@
import { IRequest } from "itty-router";
import { Constants } from "../../constants";
import { twitterFetch } from "../../fetch";
import { buildAPITweet } from "./processor";
import { Experiment, experimentCheck } from "../../experiments";
import { isGraphQLTweet } from "../../helpers/graphql";
import { IRequest } from 'itty-router';
import { Constants } from '../../constants';
import { twitterFetch } from '../../fetch';
import { buildAPITweet } from './processor';
import { Experiment, experimentCheck } from '../../experiments';
import { isGraphQLTweet } from '../../helpers/graphql';
export const fetchTweetDetail = async (
status: string,
event: FetchEvent,
useElongator = typeof TwitterProxy !== 'undefined',
cursor: string | null = null
): Promise<GraphQLTweetFoundResponse> => {
): Promise<TweetDetailResult> => {
return (await twitterFetch(
`${
Constants.TWITTER_ROOT
}/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent(
JSON.stringify({
focalTweetId: status,
referrer: "home",
referrer: 'home',
with_rux_injections: false,
includePromotedContent: false,
withCommunity: true,
@ -58,16 +58,33 @@ export const fetchTweetDetail = async (
event,
useElongator,
(_conversation: unknown) => {
const conversation = _conversation as GraphQLTweetFoundResponse;
const tweet = findTweetInBucket(status, processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions));
const conversation = _conversation as TweetDetailResult;
const tweet = findTweetInBucket(
status,
processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions)
);
if (tweet && isGraphQLTweet(tweet)) {
return true;
}
console.log('invalid graphql tweet');
console.log('invalid graphql tweet', conversation);
const firstInstruction = (
conversation.data?.threaded_conversation_with_injections_v2
.instructions?.[0] as TimelineAddEntriesInstruction
)?.entries?.[0];
if (
(
(firstInstruction as { content: GraphQLTimelineItem })?.content
?.itemContent as GraphQLTimelineTweet
)?.tweet_results?.result?.__typename === 'TweetTombstone'
) {
console.log('tweet is private');
return true;
}
return Array.isArray(conversation?.errors);
}
)) as GraphQLTweetFoundResponse;
},
true
)) as TweetDetailResult;
};
export const fetchByRestId = async (
@ -147,11 +164,11 @@ export const fetchByRestId = async (
}
// Final clause for checking if it's valid is if there's errors
return Array.isArray(conversation.errors);
}
},
false
)) as TweetResultsByRestIdResult;
};
const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => {
const bucket: GraphQLProcessBucket = {
tweets: [],
@ -160,13 +177,17 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke
instructions.forEach?.(instruction => {
if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') {
// @ts-expect-error Use entries or moduleItems depending on the type
(instruction?.entries ?? instruction.moduleItems).forEach((_entry) => {
const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry
const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content;
(instruction?.entries ?? instruction.moduleItems).forEach(_entry => {
const entry = _entry as
| GraphQLTimelineTweetEntry
| GraphQLConversationThread
| GraphQLModuleTweetEntry;
const content =
(entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content;
if (content.__typename === 'TimelineTimelineItem') {
const itemContentType = content.itemContent?.__typename;
if (itemContentType === 'TimelineTweet') {
const entryType = content.itemContent.tweet_results.result.__typename
const entryType = content.itemContent.tweet_results.result.__typename;
if (entryType === 'Tweet') {
bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet);
}
@ -176,16 +197,20 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke
} else if (itemContentType === 'TimelineTimelineCursor') {
bucket.cursors.push(content.itemContent as GraphQLTimelineCursor);
}
} else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') {
content.items.forEach((item) => {
} else if (
(content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule'
) {
content.items.forEach(item => {
const itemContentType = item.item.itemContent.__typename;
if (itemContentType === 'TimelineTweet') {
const entryType = item.item.itemContent.tweet_results.result.__typename
const entryType = item.item.itemContent.tweet_results.result.__typename;
if (entryType === 'Tweet') {
bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet);
}
if (entryType === 'TweetWithVisibilityResults') {
bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet);
bucket.tweets.push(
item.item.itemContent.tweet_results.result.tweet as GraphQLTweet
);
}
} else if (itemContentType === 'TimelineTimelineCursor') {
bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor);
@ -194,18 +219,18 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke
}
});
}
})
});
return bucket;
}
};
const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => {
return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null;
}
};
const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => {
return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id);
}
};
const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => {
const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id);
@ -213,10 +238,15 @@ const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number =>
console.log('uhhh, we could not even find that tweet, dunno how that happened');
return -1;
}
return bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str);
}
return bucket.tweets.findIndex(
_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str
);
};
const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => {
const consolidateCursors = (
oldCursors: GraphQLTimelineCursor[],
newCursors: GraphQLTimelineCursor[]
): GraphQLTimelineCursor[] => {
/* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */
return oldCursors.map(cursor => {
const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType);
@ -225,58 +255,87 @@ const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: Gra
}
return cursor;
});
}
};
const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => {
return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id)
}
return tweets.filter(
tweet =>
tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id
);
};
export const constructTwitterThread = async (id: string,
/* Fetch and construct a Twitter thread */
export const constructTwitterThread = async (
id: string,
processThread = false,
request: IRequest,
language: string | undefined,
legacyAPI = false): Promise<SocialThread> => {
legacyAPI = false
): Promise<SocialThread> => {
console.log('legacyAPI', legacyAPI);
console.log('legacyAPI', legacyAPI)
let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult;
let response: TweetDetailResult | TweetResultsByRestIdResult | null = null;
let post: APITweet;
if (typeof TwitterProxy === "undefined" || !experimentCheck(Experiment.TWEET_DETAIL_API)) {
console.log('Using TweetResultsByRestId for request...');
response = await fetchByRestId(id, request.event) as TweetResultsByRestIdResult;
const result = response?.data?.tweetResult?.result as GraphQLTweet;
if (typeof result === "undefined") {
return { post: null, thread: null, author: null, code: 404 };
}
post = await buildAPITweet(result, language, false, legacyAPI) as APITweet;
if (post === null) {
return { post: null, thread: null, author: null, code: 404 };
}
return { post: post, thread: null, author: post.author, code: 200 };
} else {
/* We can use TweetDetail on elongator accounts to increase per-account rate limit.
We also use TweetDetail to process threads (WIP) */
if (typeof TwitterProxy !== 'undefined' && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) {
console.log('Using TweetDetail for request...');
response = await fetchTweetDetail(id, request.event) as GraphQLTweetFoundResponse;
response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult;
if (!response.data) {
console.log(response);
const firstInstruction = (
response.data?.threaded_conversation_with_injections_v2
.instructions?.[0] as TimelineAddEntriesInstruction
)?.entries?.[0];
if (
(
(firstInstruction as { content: GraphQLTimelineItem })?.content
?.itemContent as GraphQLTimelineTweet
)?.tweet_results?.result?.__typename === 'TweetTombstone' /* If a tweet is private */
) {
console.log('tweet is private');
return { post: null, thread: null, author: null, code: 401 };
} else if (!response.data) {
return { post: null, thread: null, author: null, code: 404 };
}
}
const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions);
/* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */
if (!response) {
console.log('Using TweetResultsByRestId for request...');
response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult;
const result = response?.data?.tweetResult?.result as GraphQLTweet;
if (typeof result === 'undefined') {
return { post: null, thread: null, author: null, code: 404 };
}
const buildPost = await buildAPITweet(result, language, false, legacyAPI);
if ((buildPost as FetchResults).status === 401) {
return { post: null, thread: null, author: null, code: 401 };
} else if (buildPost === null) {
return { post: null, thread: null, author: null, code: 404 };
}
post = buildPost as APITweet;
return { post: post, thread: null, author: post.author, code: 200 };
}
const bucket = processResponse(
response.data.threaded_conversation_with_injections_v2.instructions
);
const originalTweet = findTweetInBucket(id, bucket);
/* Don't bother processing thread on a null tweet */
if (originalTweet === null) {
return { post: null, thread: null, author: null, code: 404 };
}
post = await buildAPITweet(originalTweet, undefined, false, legacyAPI) as APITweet;
post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet;
if (post === null) {
return { post: null, thread: null, author: null, code: 404 };
@ -301,41 +360,58 @@ export const constructTwitterThread = async (id: string,
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
console.log(
'adding next tweet to thread',
newCurrentId,
'from',
currentId,
'at index',
index,
'in bucket'
);
threadTweets.push(tweet);
currentId = newCurrentId;
console.log('Current index', index, 'of', bucket.tweets.length)
console.log('Current index', index, 'of', bucket.tweets.length);
/* Reached the end of the current list of tweets in thread) */
if (index >= (bucket.tweets.length - 1)) {
if (index >= bucket.tweets.length - 1) {
/* See if we have a cursor to fetch more tweets */
const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore'));
console.log('current cursors: ', bucket.cursors)
const cursor = bucket.cursors.find(
cursor => cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore'
);
console.log('current cursors: ', bucket.cursors);
if (!cursor) {
console.log('No cursor present, stopping pagination down')
console.log('No cursor present, stopping pagination down');
break;
}
console.log('Cursor present, fetching more tweets down');
let loadCursor: GraphQLTweetFoundResponse;
let loadCursor: TweetDetailResult;
try {
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value)
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value);
if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
if (
typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions ===
'undefined'
) {
console.log('Unknown data while fetching cursor', loadCursor);
break;
}
} catch(e) {
} catch (e) {
console.log('Error fetching cursor', e);
break;
}
const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet));
const cursorResponse = processResponse(
loadCursor.data.threaded_conversation_with_injections_v2.instructions
);
bucket.tweets = bucket.tweets.concat(
filterBucketTweets(cursorResponse.tweets, originalTweet)
);
/* Remove old cursor and add new bottom cursor if necessary */
consolidateCursors(bucket.cursors, cursorResponse.cursors);
console.log('updated bucket of cursors', bucket.cursors);
@ -351,36 +427,54 @@ export const constructTwitterThread = async (id: string,
const tweet = bucket.tweets[index];
const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str;
console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket')
console.log(
'adding previous tweet to thread',
newCurrentId,
'from',
currentId,
'at index',
index,
'in bucket'
);
threadTweets.unshift(tweet);
currentId = newCurrentId;
if (index === 0) {
/* See if we have a cursor to fetch more tweets */
const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore'));
console.log('current cursors: ', bucket.cursors)
const cursor = bucket.cursors.find(
cursor => cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore'
);
console.log('current cursors: ', bucket.cursors);
if (!cursor) {
console.log('No cursor present, stopping pagination up')
console.log('No cursor present, stopping pagination up');
break;
}
console.log('Cursor present, fetching more tweets up');
let loadCursor: GraphQLTweetFoundResponse;
let loadCursor: TweetDetailResult;
try {
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value)
loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value);
if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') {
if (
typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions ===
'undefined'
) {
console.log('Unknown data while fetching cursor', loadCursor);
break;
}
} catch(e) {
} catch (e) {
console.log('Error fetching cursor', e);
break;
}
const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions);
bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet));
const cursorResponse = processResponse(
loadCursor.data.threaded_conversation_with_injections_v2.instructions
);
bucket.tweets = cursorResponse.tweets.concat(
filterBucketTweets(bucket.tweets, originalTweet)
);
/* Remove old cursor and add new top cursor if necessary */
consolidateCursors(bucket.cursors, cursorResponse.cursors);
@ -396,14 +490,14 @@ export const constructTwitterThread = async (id: string,
thread: [],
author: author,
code: 200
}
};
threadTweets.forEach(async (tweet) => {
socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false) as APITweet);
threadTweets.forEach(async tweet => {
socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet);
});
return socialThread;
}
};
export const threadAPIProvider = async (request: IRequest) => {
const { id } = request.params;
@ -412,5 +506,5 @@ export const threadAPIProvider = async (request: IRequest) => {
return new Response(JSON.stringify(processedResponse), {
headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS }
})
}
});
};

37
src/providers/twitter/processor.ts
View file

@ -14,7 +14,7 @@ export const buildAPITweet = async (
threadPiece = false,
legacyAPI = false
// eslint-disable-next-line sonarjs/cognitive-complexity
): Promise<APITweet | null> => {
): Promise<APITweet | FetchResults | null> => {
const apiTweet = {} as APITweet;
/* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'.
@ -38,7 +38,11 @@ export const buildAPITweet = async (
if (typeof tweet.core === 'undefined') {
console.log('Tweet still not valid', tweet);
return null;
if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') {
return { status: 401 };
} else {
return { status: 404 };
}
}
const graphQLUser = tweet.core.user_results.result;
@ -50,7 +54,9 @@ export const buildAPITweet = async (
/* Populating a lot of the basics */
apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`;
apiTweet.id = id;
apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || ''));
apiTweet.text = unescapeText(
linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')
);
if (!threadPiece) {
apiTweet.author = {
id: apiUser.id,
@ -75,7 +81,7 @@ export const buildAPITweet = async (
if (legacyAPI) {
// @ts-expect-error Use retweets for legacy API
apiTweet.retweets = tweet.legacy.retweet_count;
// @ts-expect-error `tweets` is only part of legacy API
apiTweet.author.tweets = apiTweet.author.posts;
// @ts-expect-error Part of legacy API that we no longer are able to track
@ -139,11 +145,16 @@ export const buildAPITweet = async (
}
apiTweet.media = {};
/* We found a quote tweet, let's process that too */
const quoteTweet = tweet.quoted_status_result;
if (quoteTweet) {
apiTweet.quote = (await buildAPITweet(quoteTweet, language, threadPiece, legacyAPI)) as APITweet;
apiTweet.quote = (await buildAPITweet(
quoteTweet,
language,
threadPiece,
legacyAPI
)) as APITweet;
/* Only override the embed_card if it's a basic tweet, since media always takes precedence */
if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) {
apiTweet.embed_card = apiTweet.quote.embed_card;
@ -211,7 +222,11 @@ export const buildAPITweet = async (
}
}
if (apiTweet.media?.videos && apiTweet.media?.videos.length > 0 && apiTweet.embed_card !== 'player') {
if (
apiTweet.media?.videos &&
apiTweet.media?.videos.length > 0 &&
apiTweet.embed_card !== 'player'
) {
apiTweet.embed_card = 'player';
}
@ -221,7 +236,9 @@ export const buildAPITweet = async (
const translateAPI = await translateTweet(tweet, '', language);
if (translateAPI !== null && translateAPI?.translation) {
apiTweet.translation = {
text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')),
text: unescapeText(
linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')
),
source_lang: translateAPI?.sourceLanguage || '',
target_lang: translateAPI?.destinationLanguage || '',
source_lang_en: translateAPI?.localizedSourceLanguage || ''
@ -233,7 +250,7 @@ export const buildAPITweet = async (
// @ts-expect-error Use twitter_card for legacy API
apiTweet.twitter_card = apiTweet.embed_card;
// @ts-expect-error Part of legacy API that we no longer are able to track
apiTweet.color = null
apiTweet.color = null;
// @ts-expect-error Use twitter_card for legacy API
delete apiTweet.embed_card;
if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) {
@ -243,4 +260,4 @@ export const buildAPITweet = async (
}
return apiTweet;
};
};

21
src/types/twitterTypes.d.ts vendored
View file

@ -355,6 +355,7 @@ type GraphQLTweet = {
// Workaround
result: GraphQLTweet;
__typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable';
reason: string; // used for errors
rest_id: string; // "1674824189176590336",
has_birdwatch_notes: false;
core: {
@ -451,14 +452,14 @@ type GraphQLTimelineTweet = {
tweet_results: {
result: GraphQLTweet | TweetTombstone;
};
}
};
type GraphQLTimelineCursor = {
cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore';
itemType: 'TimelineTimelineCursor';
value: string;
__typename: 'TimelineTimelineCursor';
}
};
interface GraphQLBaseTimeline {
entryType: string;
@ -469,16 +470,16 @@ type GraphQLTimelineItem = GraphQLBaseTimeline & {
entryType: 'TimelineTimelineItem';
__typename: 'TimelineTimelineItem';
itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor;
}
};
type GraphQLTimelineModule = GraphQLBaseTimeline & {
entryType: 'TimelineTimelineModule';
__typename: 'TimelineTimelineModule';
items: {
entryId: `conversationthread-${number}-tweet-${number}`;
item: GraphQLTimelineItem
item: GraphQLTimelineItem;
}[];
}
};
type GraphQLTimelineTweetEntry = {
/** The entryID contains the tweet ID */
@ -501,7 +502,10 @@ type GraphQLConversationThread = {
type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown;
type ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction;
type ThreadInstruction =
| TimelineAddEntriesInstruction
| TimelineTerminateTimelineInstruction
| TimelineAddModulesInstruction;
type TimelineAddEntriesInstruction = {
type: 'TimelineAddEntries';
@ -543,7 +547,7 @@ type GraphQLTweetNotFoundResponse = {
];
data: Record<string, never>;
};
type GraphQLTweetFoundResponse = {
type TweetDetailResult = {
errors?: unknown[];
data: {
threaded_conversation_with_injections_v2: {
@ -564,8 +568,7 @@ type TweetResultsByRestIdResult = {
type TweetStub = {
__typename: 'TweetUnavailable';
reason: 'NsfwLoggedOut' | 'Protected';
}
};
interface GraphQLProcessBucket {
tweets: GraphQLTweet[];

8
src/types/types.d.ts vendored
View file

@ -144,7 +144,7 @@ interface APIPost {
replying_to: {
screen_name: string | null;
post: string | null;
} | null
} | null;
source: string | null;
@ -197,4 +197,8 @@ interface SocialThread {
thread: (APIPost | APITweet)[] | null;
author: APIUser | null;
code: number;
}
}
interface FetchResults {
status: number;
}

4
src/worker.ts
View file

@ -10,7 +10,7 @@ import { Strings } from './strings';
import motd from '../motd.json';
import { sanitizeText } from './helpers/utils';
import { handleProfile } from './user';
import { threadAPIProvider } from './providers/twitter/conversation';
// import { threadAPIProvider } from './providers/twitter/conversation';
declare const globalThis: {
fetchCompletedTime: number;
@ -454,7 +454,7 @@ router.get('/status/:id', statusRequest);
router.get('/status/:id/:language', statusRequest);
router.get('/version', versionRequest);
router.get('/set_base_redirect', setRedirectRequest);
router.get('/v2/twitter/thread/:id', threadAPIProvider)
// router.get('/v2/twitter/thread/:id', threadAPIProvider)
/* Oembeds (used by Discord to enhance responses)