diff --git a/package-lock.json b/package-lock.json index 7849511..5bfdc3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1735,9 +1735,9 @@ "dev": true }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.19", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", - "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz", + "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==", "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -2038,14 +2038,14 @@ } }, "node_modules/@sentry-internal/tracing": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.0.tgz", - "integrity": "sha512-JK6IRGgdtZjswGfaGIHNWIThffhOHzVIIaGmglui+VFIzOsOqePjoxaDV0MEvzafxXZD7eWqGE5RGuZ0n6HFVg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.1.tgz", + "integrity": "sha512-nNaiZreQxCitG2PzYPaC7XtyA9OMsETGYMKAtiK4p62/uTmeYbsBva9BoNx1XeiHRwbrVQYRMKQ9nV5e2jS4/A==", "dev": true, "dependencies": { - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2092,13 +2092,13 @@ } }, "node_modules/@sentry/core": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.0.tgz", - "integrity": "sha512-83NRuqn7nDZkSVBN5yJQqcpXDG4yMYiB7TkYUKrGTzBpRy6KUOrkCdybuKk0oraTIGiGSe5WEwCFySiNgR9FzA==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.1.tgz", + "integrity": "sha512-LvEhOSfdIvwkr+PdlrT/aA/iOLhkXrSkvjqAQyogE4ddCWeYfS0NoirxNt1EaxMBAWKhYZRqzkA7WA4LDLbzlA==", "dev": true, "dependencies": { - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2172,15 +2172,15 @@ } }, "node_modules/@sentry/node": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.0.tgz", - "integrity": "sha512-uBmW2/z0cz/WFIG74ZF7lSipO0XNzMf9yrdqnZXnGDYsUZE4I4QiqDN0hNi6fkTgf9MYRC8uFem2OkAvyPJ74Q==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.1.tgz", + "integrity": "sha512-aMUQ2LFZF64FBr+cgjAqjT4OkpYBIC9lyWI8QqjEHqNho5+LGu18/iVrJPD4fgs4UhGdCuAiQjpC36MbmnIDZA==", "dev": true, "dependencies": { - "@sentry-internal/tracing": "7.74.0", - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry-internal/tracing": "7.74.1", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "cookie": "^0.5.0", "https-proxy-agent": "^5.0.0", "lru_map": "^0.3.3", @@ -2191,21 +2191,21 @@ } }, "node_modules/@sentry/types": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.0.tgz", - "integrity": "sha512-rI5eIRbUycWjn6s6o3yAjjWtIvYSxZDdnKv5je2EZINfLKcMPj1dkl6wQd2F4y7gLfD/N6Y0wZYIXC3DUdJQQg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.1.tgz", + "integrity": "sha512-2jIuPc+YKvXqZETwr2E8VYnsH1zsSUR/wkIvg1uTVeVNyoowJv+YsOtCdeGyL2AwiotUBSPKu7O1Lz0kq5rMOQ==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/@sentry/utils": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.0.tgz", - "integrity": "sha512-k3np8nuTPtx5KDODPtULfFln4UXdE56MZCcF19Jv6Ljxf+YN/Ady1+0Oi3e0XoSvFpWNyWnglauT7M65qCE6kg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.1.tgz", + "integrity": "sha512-qUsqufuHYcy5gFhLZslLxA5kcEOkkODITXW3c7D+x+8iP/AJqa8v8CeUCVNS7RetHCuIeWAbbTClC4c411EwQg==", "dev": true, "dependencies": { - "@sentry/types": "7.74.0", + "@sentry/types": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2237,9 +2237,9 @@ } }, "node_modules/@types/babel__core": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.2.tgz", - "integrity": "sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA==", + "version": "7.20.3", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.3.tgz", + "integrity": "sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA==", "dev": true, "dependencies": { "@babel/parser": "^7.20.7", @@ -2250,18 +2250,18 @@ } }, "node_modules/@types/babel__generator": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.5.tgz", - "integrity": "sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.6.tgz", + "integrity": "sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w==", "dev": true, "dependencies": { "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__template": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.2.tgz", - "integrity": "sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.3.tgz", + "integrity": "sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ==", "dev": true, "dependencies": { "@babel/parser": "^7.1.0", @@ -2269,18 +2269,18 @@ } }, "node_modules/@types/babel__traverse": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.2.tgz", - "integrity": "sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw==", + "version": "7.20.3", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.3.tgz", + "integrity": "sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw==", "dev": true, "dependencies": { "@babel/types": "^7.20.7" } }, "node_modules/@types/better-sqlite3": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.5.tgz", - "integrity": "sha512-H3ZUx89KiPhYa9nalUXVVStSUFHuzYxt4yoazufpTTYW9rVUCzhh02V8CH2C8nE4libnK0UgFq5DFIe0DOhqow==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.6.tgz", + "integrity": "sha512-nuFAptzt0hZYBvyLzKQCbuCCK+RN9PHH4ezar5EJLIg2qpVhwQ/uLvLO/K8A9O7N8DafawgFupiyXQSs0U48Ng==", "dev": true, "dependencies": { "@types/node": "*" @@ -3536,9 +3536,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.554", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.554.tgz", - "integrity": "sha512-Q0umzPJjfBrrj8unkONTgbKQXzXRrH7sVV7D9ea2yBV3Oaogz991yhbpfvo2LMNkJItmruXTEzVpP9cp7vaIiQ==", + "version": "1.4.557", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.557.tgz", + "integrity": "sha512-6x0zsxyMXpnMJnHrondrD3SuAeKcwij9S+83j2qHAQPXbGTDDfgImzzwgGlzrIcXbHQ42tkG4qA6U860cImNhw==", "dev": true }, "node_modules/emittery": { diff --git a/src/api/status.ts b/src/api/status.ts index 569b8c1..7913b0a 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -61,6 +61,7 @@ const populateTweetProperties = async ( name: apiUser.name, screen_name: apiUser.screen_name, avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', + // @ts-expect-error Legacy api shit avatar_color: null, banner_url: apiUser.banner_url || '', description: apiUser.description || '', @@ -77,8 +78,11 @@ const populateTweetProperties = async ( }; apiTweet.replies = tweet.legacy.reply_count; apiTweet.retweets = tweet.legacy.retweet_count; + apiTweet.reposts = tweet.legacy.retweet_count; apiTweet.likes = tweet.legacy.favorite_count; + // @ts-expect-error Legacy api shit apiTweet.color = null; + // @ts-expect-error legacy api apiTweet.twitter_card = 'tweet'; apiTweet.created_at = tweet.legacy.created_at; apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; @@ -131,11 +135,11 @@ const populateTweetProperties = async ( apiTweet.media.all.push(mediaObject); if (mediaObject.type === 'photo') { - apiTweet.twitter_card = 'summary_large_image'; + apiTweet.embed_card = 'summary_large_image'; apiTweet.media.photos = apiTweet.media.photos || []; apiTweet.media.photos.push(mediaObject); } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { - apiTweet.twitter_card = 'player'; + apiTweet.embed_card = 'player'; apiTweet.media.videos = apiTweet.media.videos || []; apiTweet.media.videos.push(mediaObject); } else { @@ -183,9 +187,9 @@ const populateTweetProperties = async ( /* Workaround: Force player card by default for videos */ /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' The mediaList however can set it to something else. TODO: Reimplement as enums */ - // @ts-expect-error see above comment - if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') { - apiTweet.twitter_card = 'player'; + + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; } /* If a language is specified in API or by user, let's try translating it! */ diff --git a/src/api/user.ts b/src/api/user.ts index 45885eb..499900c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.following = user.legacy.friends_count; apiUser.likes = user.legacy.favourites_count; apiUser.tweets = user.legacy.statuses_count; + apiUser.posts = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description); - apiUser.location = user.legacy.location; - apiUser.banner_url = user.legacy.profile_banner_url; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` + apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null; + apiUser.location = user.legacy.location ? user.legacy.location : null; + apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/constants.ts b/src/constants.ts index 5c8ff7e..addbee0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,7 @@ export const Constants = { RELEASE_NAME: RELEASE_NAME, API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`, TWITTER_ROOT: 'https://twitter.com', + TWITTER_GLOBAL_NAME_ROOT: 'twitter.com', TWITTER_API_ROOT: 'https://api.twitter.com', BOT_UA_REGEX: /bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi, diff --git a/src/embed/status.ts b/src/embed/status.ts index 8c4f413..a078ca6 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -120,7 +120,7 @@ export const handleStatus = async ( const headers = [ ``, ``, - ``, + ``, ``, ``, `` @@ -199,8 +199,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card === 'player') { - tweet.twitter_card = 'summary_large_image'; + if (tweet.embed_card === 'player') { + tweet.embed_card = 'summary_large_image'; } break; case 'video': @@ -216,8 +216,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card !== 'player') { - tweet.twitter_card = 'player'; + if (tweet.embed_card !== 'player') { + tweet.embed_card = 'player'; } /* This Tweet has a video to render. */ break; @@ -344,7 +344,7 @@ export const handleStatus = async ( and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */ const text = useIV ? sanitizeText(newText).replace(/\n/g, '
') : sanitizeText(newText); - const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card; + const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card; /* Push basic headers relating to author, Tweet text, and site name */ headers.push( diff --git a/src/helpers/useragent.ts b/src/helpers/useragent.ts index 512d9bb..d09d536 100644 --- a/src/helpers/useragent.ts +++ b/src/helpers/useragent.ts @@ -1,6 +1,6 @@ /* We keep this value up-to-date for making our requests to Twitter as indistinguishable from normal user traffic as possible. */ -const fakeChromeVersion = 116; +const fakeChromeVersion = 118; const platformWindows = 'Windows NT 10.0; Win64; x64'; const platformMac = 'Macintosh; Intel Mac OS X 10_15_7'; const platformLinux = 'X11; Linux x86_64'; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts new file mode 100644 index 0000000..3b8e1b7 --- /dev/null +++ b/src/providers/twitter/processor.ts @@ -0,0 +1,226 @@ +import { renderCard } from '../../helpers/card'; +import { Constants } from '../../constants'; +import { linkFixer } from '../../helpers/linkFixer'; +import { handleMosaic } from '../../helpers/mosaic'; +// import { translateTweet } from '../../helpers/translate'; +import { unescapeText } from '../../helpers/utils'; +import { processMedia } from '../../helpers/media'; +import { convertToApiUser } from '../../api/user'; + +export const buildAPITweet = async ( + tweet: GraphQLTweet, + language: string | undefined, + threadPiece = false, + legacyAPI = false + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const apiTweet = {} as APITweet; + + /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. + It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ + + if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { + tweet = tweet.result; + } + + if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { + tweet.core = tweet.tweet.core; + } + + if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { + tweet.legacy = tweet.tweet?.legacy; + } + + if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { + tweet.views = tweet?.tweet?.views; + } + + if (typeof tweet.core === 'undefined') { + console.log('Tweet still not valid', tweet); + return null; + } + + const graphQLUser = tweet.core.user_results.result; + const apiUser = convertToApiUser(graphQLUser); + + /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ + const id = tweet.rest_id ?? tweet.legacy.id_str; + + /* Populating a lot of the basics */ + apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; + apiTweet.id = id; + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); + if (!threadPiece) { + apiTweet.author = { + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, + global_screen_name: apiUser.global_screen_name, + avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, + banner_url: apiUser.banner_url ?? null, + description: apiUser.description ?? null, + location: apiUser.location ?? null, + url: apiUser.url ?? null, + followers: apiUser.followers, + following: apiUser.following, + joined: apiUser.joined, + posts: apiUser.tweets, + likes: apiUser.likes, + protected: apiUser.protected, + birthday: apiUser.birthday, + website: apiUser.website + }; + } + apiTweet.replies = tweet.legacy.reply_count; + if (legacyAPI) { + apiTweet.retweets = tweet.legacy.retweet_count; + } else { + apiTweet.reposts = tweet.legacy.retweet_count; + } + apiTweet.likes = tweet.legacy.favorite_count; + apiTweet.embed_card = 'tweet'; + apiTweet.created_at = tweet.legacy.created_at; + apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; + + apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; + + if (tweet.views.state === 'EnabledWithCount') { + apiTweet.views = parseInt(tweet.views.count || '0') ?? null; + } else { + apiTweet.views = null; + } + console.log('note_tweet', JSON.stringify(tweet.note_tweet)); + const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; + + if (noteTweetText) { + tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; + tweet.legacy.entities.hashtags = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; + tweet.legacy.entities.symbols = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; + + console.log('We meet the conditions to use new note tweets'); + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); + apiTweet.is_note_tweet = true; + } else { + apiTweet.is_note_tweet = false; + } + + if (tweet.legacy.lang !== 'unk') { + apiTweet.lang = tweet.legacy.lang; + } else { + apiTweet.lang = null; + } + + if (legacyAPI) { + apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; + } else if (tweet.legacy.in_reply_to_screen_name) { + apiTweet.reply_of = { + screen_name: tweet.legacy.in_reply_to_screen_name || null, + post: tweet.legacy.in_reply_to_status_id_str || null + }; + } else { + apiTweet.reply_of = null; + } + + apiTweet.media = { + all: [], + photos: [], + videos: [], + }; + + /* We found a quote tweet, let's process that too */ + const quoteTweet = tweet.quoted_status_result; + if (quoteTweet) { + apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet'&& apiTweet.quote !== null) { + apiTweet.embed_card = apiTweet.quote.embed_card; + } + } + + const mediaList = Array.from( + tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] + ); + + // console.log('tweet', JSON.stringify(tweet)); + + /* Populate this Tweet's media */ + mediaList.forEach(media => { + const mediaObject = processMedia(media); + if (mediaObject) { + apiTweet.media?.all?.push(mediaObject); + if (mediaObject.type === 'photo') { + apiTweet.embed_card = 'summary_large_image'; + apiTweet.media?.photos?.push(mediaObject); + } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { + apiTweet.embed_card = 'player'; + apiTweet.media?.videos?.push(mediaObject); + } else { + console.log('Unknown media type', mediaObject.type); + } + } + }); + + /* Grab color palette data */ + /* + if (mediaList[0]?.ext_media_color?.palette) { + apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); + } + */ + + /* Handle photos and mosaic if available */ + if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) { + const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); + if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { + apiTweet.media.mosaic = mosaic; + } + } + + // Add Tweet source but remove the link HTML tag + if (tweet.source) { + apiTweet.source = (tweet.source || '').replace( + /(.+?)<\/a>/, + '$2' + ); + } + + /* Populate a Twitter card */ + + if (tweet.card) { + const card = renderCard(tweet.card); + if (card.external_media) { + apiTweet.media = apiTweet.media ?? {}; + apiTweet.media.external = card.external_media; + } + if (card.poll) { + apiTweet.poll = card.poll; + } + } + + /* Workaround: Force player card by default for videos */ + /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' + The mediaList however can set it to something else. TODO: Reimplement as enums */ + // @ts-expect-error see above comment + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; + } + + /* If a language is specified in API or by user, let's try translating it! */ + if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { + /* TODO: Reimplement */ + // console.log(`Attempting to translate Tweet to ${language}...`); + // const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); + // if (translateAPI !== null && translateAPI?.translation) { + // apiTweet.translation = { + // text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + // source_lang: translateAPI?.sourceLanguage || '', + // target_lang: translateAPI?.destinationLanguage || '', + // source_lang_en: translateAPI?.localizedSourceLanguage || '' + // }; + // } + } + + return apiTweet; +}; \ No newline at end of file diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/status.ts new file mode 100644 index 0000000..f43c141 --- /dev/null +++ b/src/providers/twitter/status.ts @@ -0,0 +1,308 @@ +import { IRequest } from "itty-router"; +import { Constants } from "../../constants"; +import { twitterFetch } from "../../fetch"; +import { buildAPITweet } from "./processor"; + +type GraphQLProcessBucket = { + tweets: GraphQLTweet[]; + cursors: GraphQLTimelineCursor[]; +} + +type SocialThread = { + post: APIPost | APITweet | null; + thread: (APIPost | APITweet)[] | null; + author: APIUser | null; +} + +export const fetchTwitterThread = async ( + status: string, + event: FetchEvent, + useElongator = typeof TwitterProxy !== 'undefined', + cursor: string | null = null +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( + JSON.stringify({ + focalTweetId: status, + referrer: "home", + with_rux_injections: false, + includePromotedContent: false, + withCommunity: true, + withBirdwatchNotes: true, + withQuickPromoteEligibilityTweetFields: false, + withVoice: false, + withV2Timeline: true, + cursor: cursor + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_home_pinned_timelines_enabled: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_media_download_video_enabled: true, + responsive_web_enhance_cards_enabled: true + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + () => { + return true; + } + )) as GraphQLTweetFoundResponse; +}; + +const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => { + const bucket: GraphQLProcessBucket = { + tweets: [], + cursors: [] + }; + instructions.forEach?.(instruction => { + if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { + // @ts-expect-error Use entries or moduleItems depending on the type + (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => { + const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry + const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + if (content.__typename === 'TimelineTimelineItem') { + const itemContentType = content.itemContent?.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = content.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); + } + } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') { + content.items.forEach((item) => { + const itemContentType = item.item.itemContent.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = item.item.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); + } + }); + } + }); + } + }) + + return bucket; +} + +const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { + return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; +} + +const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { + return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); +} + +const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { + const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); + if (!tweet) { + console.log('uhhh, we could not even find that tweet, dunno how that happened'); + return -1; + } + const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); + if (index === -1) { + console.log('could not find shit for', id) + console.log(bucket.cursors) + } + return index; +} + +const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { + /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ + return oldCursors.map(cursor => { + const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); + if (newCursor) { + return newCursor; + } + return cursor; + }); +} + +const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { + return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) +} + +export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise => { + const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse; + + if (!response.data) { + return { post: null, thread: null, author: null }; + } + + const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); + const originalTweet = findTweetInBucket(id, bucket); + + /* Don't bother processing thread on a null tweet */ + if (originalTweet === null) { + return { post: null, thread: null, author: null }; + } + + const post = await buildAPITweet(originalTweet, undefined, false, false); + const author = post.author; + /* remove post.author */ + // @ts-expect-error lmao + delete post.author; + + /* If we're not processing threads, let's be done here */ + if (!processThread) { + return { post: post, thread: null, author: author }; + } + + const threadTweets = [originalTweet]; + bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet); + + let currentId = id; + + /* Process tweets that are following the current one in the thread */ + while (findNextTweet(currentId, bucket) !== -1) { + const index = findNextTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.push(tweet); + + currentId = newCurrentId; + + console.log('Current index', index, 'of', bucket.tweets.length) + + /* Reached the end of the current list of tweets in thread) */ + if (index >= (bucket.tweets.length - 1)) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination down') + break; + } + console.log('Cursor present, fetching more tweets down'); + + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet)); + /* Remove old cursor and add new bottom cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of next tweet:', findNextTweet(currentId, bucket)); + } + + currentId = id; + + while (findPreviousTweet(currentId, bucket) !== -1) { + const index = findPreviousTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.unshift(tweet); + + currentId = newCurrentId; + + if (index === 0) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination up') + break; + } + console.log('Cursor present, fetching more tweets up'); + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet)); + /* Remove old cursor and add new top cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + + // console.log('updated bucket of tweets', bucket.tweets); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket)); + } + + const socialThread: SocialThread = { + post: post, + thread: [], + author: author + } + + threadTweets.forEach(async (tweet) => { + socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false)); + }); + + return socialThread; +} + +export const threadAPIProvider = async (request: IRequest) => { + const { id } = request.params; + + const processedResponse = await processTwitterThread(id, true, request); + + return new Response(JSON.stringify(processedResponse), { + headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } + }) +} \ No newline at end of file diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 35bce9b..8757117 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -354,7 +354,7 @@ type GraphQLTweetLegacy = { type GraphQLTweet = { // Workaround result: GraphQLTweet; - __typename: 'Tweet' | 'TweetUnavailable'; + __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -444,37 +444,76 @@ type TweetTombstone = { }; }; }; + +type GraphQLTimelineTweet = { + item: 'TimelineTweet'; + __typename: 'TimelineTweet'; + tweet_results: { + result: GraphQLTweet | TweetTombstone; + }; +} + +type GraphQLTimelineCursor = { + cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; + itemType: 'TimelineTimelineCursor'; + value: string; + __typename: 'TimelineTimelineCursor'; +} + +interface GraphQLBaseTimeline { + entryType: string; + __typename: string; +} + +type GraphQLTimelineItem = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineItem'; + __typename: 'TimelineTimelineItem'; + itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; +} + +type GraphQLTimelineModule = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineModule'; + __typename: 'TimelineTimelineModule'; + items: { + entryId: `conversationthread-${number}-tweet-${number}`; + item: GraphQLTimelineItem + }[]; +} + type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ entryId: `tweet-${number}`; // "tweet-1674824189176590336" sortIndex: string; - content: { - entryType: 'TimelineTimelineItem'; - __typename: 'TimelineTimelineItem'; - itemContent: { - item: 'TimelineTweet'; - __typename: 'TimelineTweet'; - tweet_results: { - result: GraphQLTweet | TweetTombstone; - }; - }; - }; + content: GraphQLTimelineItem; }; + +type GraphQLModuleTweetEntry = { + /** The entryID contains the tweet ID */ + sortIndex: string; + item: GraphQLTimelineItem | GraphQLTimelineModule; +}; + type GraphQLConversationThread = { entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336" sortIndex: string; + content: GraphQLTimelineModule; }; type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction; +type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; -type TimeLineAddEntriesInstruction = { +type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; entries: GraphQLTimelineEntry[]; }; -type TimeLineTerminateTimelineInstruction = { +type TimelineAddModulesInstruction = { + type: 'TimelineAddToModule'; + moduleItems: GraphQLTimelineEntry[]; +}; + +type TimelineTerminateTimelineInstruction = { type: 'TimelineTerminateTimeline'; direction: 'Top'; }; diff --git a/src/types/types.d.ts b/src/types/types.d.ts index d6f9f7f..e911d81 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -81,14 +81,6 @@ interface APITranslate { target_lang: string; } -interface BaseUser { - id?: string; - name?: string; - screen_name?: string; - avatar_url?: string; - banner_url?: string; -} - interface APIExternalMedia { type: 'video'; url: string; @@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia { }; } -interface APITweet { +interface APIPost { id: string; url: string; text: string; @@ -144,15 +136,11 @@ interface APITweet { created_timestamp: number; likes: number; - retweets: number; + reposts: number; replies: number; - views?: number | null; - color: string | null; - - quote?: APITweet; + quote?: APIPost; poll?: APIPoll; - translation?: APITranslate; author: APIUser; media?: { @@ -169,24 +157,41 @@ interface APITweet { replying_to: string | null; replying_to_status: string | null; - source: string; + reply_of: { + screen_name: string | null; + post: string | null; + } | null - is_note_tweet: boolean; + source: string | null; - twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; + embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; } -interface APIUser extends BaseUser { +interface APITweet extends APIPost { + retweets: number; + views?: number | null; + translation?: APITranslate; + + is_note_tweet: boolean; +} + +interface APIUser { + id: string; + name: string; + screen_name: string; + global_screen_name: string; + avatar_url: string | null; + banner_url: string | null; // verified: 'legacy' | 'blue'| 'business' | 'government'; // verified_label: string; - description: string; - location: string; + description: string | null; + location: string | null; url: string; - avatar_color?: string | null; protected: boolean; followers: number; following: number; - tweets: number; + tweets?: number; + posts?: number; likes: number; joined: string; website: { diff --git a/src/worker.ts b/src/worker.ts index 0629235..0565029 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,6 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; +import { threadAPIProvider } from './providers/twitter/status'; declare const globalThis: { fetchCompletedTime: number; @@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); +router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses)