From 043f6b6e0c00864d649a67a4f1477cdef3e75a88 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Mon, 15 Aug 2022 18:38:52 -0400 Subject: [PATCH] Re-organizing stuff! Better comments! --- src/api.ts | 59 +++++-------- src/constants.ts | 3 + src/fetch.ts | 17 ++-- src/{ => helpers}/author.ts | 1 + src/{ => helpers}/card.ts | 7 +- src/{ => helpers}/linkFixer.ts | 5 +- src/helpers/media.ts | 26 ++++++ src/{ => helpers}/mosaic.ts | 9 +- src/{ => helpers}/palette.ts | 6 +- src/{pollHelper.ts => helpers/pollTime.ts} | 4 +- src/{ => helpers}/quote.ts | 3 +- src/{ => helpers}/translate.ts | 3 +- src/{ => helpers}/utils.ts | 0 src/status.ts | 85 ++++++++++++++----- src/strings.ts | 6 +- src/{ => types}/env.d.ts | 0 .../twitterTypes.d.ts} | 6 +- src/{ => types}/types.d.ts | 3 +- 18 files changed, 156 insertions(+), 87 deletions(-) rename src/{ => helpers}/author.ts (84%) rename src/{ => helpers}/card.ts (93%) rename src/{ => helpers}/linkFixer.ts (60%) create mode 100644 src/helpers/media.ts rename src/{ => helpers}/mosaic.ts (92%) rename src/{ => helpers}/palette.ts (80%) rename src/{pollHelper.ts => helpers/pollTime.ts} (94%) rename src/{ => helpers}/quote.ts (82%) rename src/{ => helpers}/translate.ts (94%) rename src/{ => helpers}/utils.ts (100%) rename src/{ => types}/env.d.ts (100%) rename src/{tweetTypes.ts => types/twitterTypes.d.ts} (96%) rename src/{ => types}/types.d.ts (94%) diff --git a/src/api.ts b/src/api.ts index 93ab764..90bb46e 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,38 +1,15 @@ -import { renderCard } from './card'; +import { renderCard } from './helpers/card'; import { Constants } from './constants'; import { fetchUsingGuest } from './fetch'; -import { linkFixer } from './linkFixer'; -import { handleMosaic } from './mosaic'; -import { colorFromPalette } from './palette'; -import { translateTweet } from './translate'; -import { unescapeText } from './utils'; - -const processMedia = (media: TweetMedia): APIPhoto | APIVideo | null => { - if (media.type === 'photo') { - return { - type: 'photo', - url: media.media_url_https, - width: media.original_info.width, - height: media.original_info.height - }; - } else if (media.type === 'video' || media.type === 'animated_gif') { - // Find the variant with the highest bitrate - const bestVariant = media.video_info?.variants?.reduce?.((a, b) => - (a.bitrate ?? 0) > (b.bitrate ?? 0) ? a : b - ); - return { - url: bestVariant?.url || '', - thumbnail_url: media.media_url_https, - duration: (media.video_info?.duration_millis || 0) / 1000, - width: media.original_info.width, - height: media.original_info.height, - format: bestVariant?.content_type || '', - type: media.type === 'animated_gif' ? 'gif' : 'video' - }; - } - return null; -}; +import { linkFixer } from './helpers/linkFixer'; +import { handleMosaic } from './helpers/mosaic'; +import { colorFromPalette } from './helpers/palette'; +import { translateTweet } from './helpers/translate'; +import { unescapeText } from './helpers/utils'; +import { processMedia } from './helpers/media'; +/* This function does the heavy lifting of processing data from Twitter API + and using it to create FixTweet's streamlined API responses */ const populateTweetProperties = async ( tweet: TweetPartial, conversation: TimelineBlobPartial, @@ -50,6 +27,7 @@ const populateTweetProperties = async ( const screenName = user?.screen_name || ''; const name = user?.name || ''; + /* Populating a lot of the basics */ apiTweet.url = `${Constants.TWITTER_ROOT}/${screenName}/status/${tweet.id_str}`; apiTweet.id = tweet.id_str; apiTweet.text = unescapeText(linkFixer(tweet, tweet.full_text)); @@ -82,6 +60,7 @@ const populateTweetProperties = async ( tweet.extended_entities?.media || tweet.entities?.media || [] ); + /* Populate this Tweet's media */ mediaList.forEach(media => { const mediaObject = processMedia(media); if (mediaObject) { @@ -99,7 +78,7 @@ const populateTweetProperties = async ( apiTweet.media.video = { // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error + // @ts-expect-error Temporary warning WARNING: 'video is deprecated and will be removed. Please use videos[0] instead.', ...mediaObject @@ -108,10 +87,12 @@ const populateTweetProperties = async ( } }); + /* Grab color palette data */ if (mediaList[0]?.ext_media_color?.palette) { apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); } + /* Handle photos and mosaic if available */ if ((apiTweet.media?.photos?.length || 0) > 1) { const mosaic = await handleMosaic(apiTweet.media?.photos || [], tweet.id_str); if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { @@ -119,6 +100,7 @@ const populateTweetProperties = async ( } } + /* Populate a Twitter card */ if (tweet.card) { const card = await renderCard(tweet.card); if (card.external_media) { @@ -131,9 +113,7 @@ const populateTweetProperties = async ( } } - console.log('language', language); - - /* If a language is specified, let's try translating it! */ + /* If a language is specified in API or by user, let's try translating it! */ if (typeof language === 'string' && language.length === 2 && language !== tweet.lang) { const translateAPI = await translateTweet( tweet, @@ -153,6 +133,9 @@ const populateTweetProperties = async ( return apiTweet; }; +/* API for Twitter statuses (Tweets) + Used internally by FixTweet's embed service, or + available for free using api.fxtwitter.com. */ export const statusAPI = async ( status: string, language: string | undefined @@ -183,6 +166,7 @@ export const statusAPI = async ( return { code: 500, message: 'API_FAIL' }; } + /* Creating the response objects */ const response: APIResponse = { code: 200, message: 'OK' } as APIResponse; const apiTweet: APITweet = (await populateTweetProperties( tweet, @@ -190,6 +174,7 @@ export const statusAPI = async ( language )) as APITweet; + /* We found a quote tweet, let's process that too */ const quoteTweet = conversation.globalObjects?.tweets?.[tweet.quoted_status_id_str || '0'] || null; if (quoteTweet) { @@ -200,6 +185,8 @@ export const statusAPI = async ( )) as APITweet; } + /* Finally, staple the Tweet to the response and return it */ response.tweet = apiTweet; + return response; }; diff --git a/src/constants.ts b/src/constants.ts index 9897caf..faf278f 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,6 +1,9 @@ +/* We keep this value up-to-date for making our requests to Twitter as + indistinguishable from normal user traffic as possible. */ const fakeChromeVersion = '104'; export const Constants = { + /* These constants are populated by variables in .env, then set by Webpack */ BRANDING_NAME: BRANDING_NAME, BRANDING_NAME_DISCORD: BRANDING_NAME_DISCORD, DIRECT_MEDIA_DOMAINS: DIRECT_MEDIA_DOMAINS.split(','), diff --git a/src/fetch.ts b/src/fetch.ts index fd28406..ef9f6c8 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -25,7 +25,12 @@ export const fetchUsingGuest = async (status: string): Promise { /* Build out reply, retweet, like counts */ if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) { diff --git a/src/card.ts b/src/helpers/card.ts similarity index 93% rename from src/card.ts rename to src/helpers/card.ts index 6de8fdc..b0e5e27 100644 --- a/src/card.ts +++ b/src/helpers/card.ts @@ -1,5 +1,6 @@ -import { calculateTimeLeftString } from './pollHelper'; +import { calculateTimeLeftString } from './pollTime'; +/* Renders card for polls and non-Twitter video embeds (i.e. YouTube) */ export const renderCard = async ( card: TweetCard ): Promise<{ poll?: APIPoll; external_media?: APIExternalMedia }> => { @@ -11,7 +12,7 @@ export const renderCard = async ( let totalVotes = 0; if (typeof values !== 'undefined') { - /* TODO: make poll code cleaner */ + /* TODO: make poll code cleaner. It really sucks. */ if ( typeof values.choice1_count !== 'undefined' && typeof values.choice2_count !== 'undefined' @@ -54,8 +55,8 @@ export const renderCard = async ( }); return { poll: poll }; - /* Oh good, a non-Twitter video URL! This enables YouTube embeds and stuff to just work */ } else if (typeof values.player_url !== 'undefined') { + /* Oh good, a non-Twitter video URL! This enables YouTube embeds and stuff to just work */ return { external_media: { type: 'video', diff --git a/src/linkFixer.ts b/src/helpers/linkFixer.ts similarity index 60% rename from src/linkFixer.ts rename to src/helpers/linkFixer.ts index 17739b2..8dfb8e7 100644 --- a/src/linkFixer.ts +++ b/src/helpers/linkFixer.ts @@ -1,10 +1,13 @@ +/* Helps replace t.co links with their originals */ export const linkFixer = (tweet: TweetPartial, text: string): string => { - // Replace t.co links with their full counterparts if (typeof tweet.entities?.urls !== 'undefined') { tweet.entities?.urls.forEach((url: TcoExpansion) => { text = text.replace(url.url, url.expanded_url); }); + /* Remove any link with unavailable original. + This means that stuff like the t.co link to pic.twitter.com + will get removed in image/video Tweets */ text = text.replace(/ ?https:\/\/t\.co\/\w{10}/g, ''); } diff --git a/src/helpers/media.ts b/src/helpers/media.ts new file mode 100644 index 0000000..811de64 --- /dev/null +++ b/src/helpers/media.ts @@ -0,0 +1,26 @@ +/* Help populate API response for media */ +export const processMedia = (media: TweetMedia): APIPhoto | APIVideo | null => { + if (media.type === 'photo') { + return { + type: 'photo', + url: media.media_url_https, + width: media.original_info.width, + height: media.original_info.height + }; + } else if (media.type === 'video' || media.type === 'animated_gif') { + /* Find the variant with the highest bitrate */ + const bestVariant = media.video_info?.variants?.reduce?.((a, b) => + (a.bitrate ?? 0) > (b.bitrate ?? 0) ? a : b + ); + return { + url: bestVariant?.url || '', + thumbnail_url: media.media_url_https, + duration: (media.video_info?.duration_millis || 0) / 1000, + width: media.original_info.width, + height: media.original_info.height, + format: bestVariant?.content_type || '', + type: media.type === 'animated_gif' ? 'gif' : 'video' + }; + } + return null; +}; diff --git a/src/mosaic.ts b/src/helpers/mosaic.ts similarity index 92% rename from src/mosaic.ts rename to src/helpers/mosaic.ts index 42c16a2..99c4c57 100644 --- a/src/mosaic.ts +++ b/src/helpers/mosaic.ts @@ -1,4 +1,4 @@ -import { Constants } from './constants'; +import { Constants } from '../constants'; export const handleMosaic = async ( mediaList: APIPhoto[], @@ -11,15 +11,13 @@ export const handleMosaic = async ( selectedDomain = domain; } - // Fallback if there are no Mosaic servers + /* Fallback if there are no Mosaic servers */ if (selectedDomain === null) { return null; } else { - // console.log('mediaList', mediaList); const mosaicMedia = mediaList.map( media => media.url?.match(/(?<=\/media\/)[\w-]+(?=[.?])/g)?.[0] || '' ); - // console.log('mosaicMedia', mosaicMedia); const baseUrl = `https://${selectedDomain}/`; let path = ''; @@ -50,7 +48,8 @@ export const handleMosaic = async ( } }; -// Port of https://github.com/FixTweet/mosaic/blob/feature/size-endpoint/src/mosaic.rs#L236 +/* TypeScript Port of https://github.com/FixTweet/mosaic/blob/feature/size-endpoint/src/mosaic.rs#L236 + We use this to generate accurate mosaic sizes which helps Discord render it correctly */ const SPACING_SIZE = 10; /* diff --git a/src/palette.ts b/src/helpers/palette.ts similarity index 80% rename from src/palette.ts rename to src/helpers/palette.ts index 85dd8a7..6c43a55 100644 --- a/src/palette.ts +++ b/src/helpers/palette.ts @@ -1,6 +1,7 @@ -import { Constants } from './constants'; +import { Constants } from '../constants'; -// https://stackoverflow.com/questions/5623838/rgb-to-hex-and-hex-to-rgb +/* Converts rgb to hex, as we use hex for API and embeds + https://stackoverflow.com/questions/5623838/rgb-to-hex-and-hex-to-rgb */ const componentToHex = (component: number) => { const hex = component.toString(16); return hex.length === 1 ? '0' + hex : hex; @@ -9,6 +10,7 @@ const componentToHex = (component: number) => { const rgbToHex = (r: number, g: number, b: number) => `#${componentToHex(r)}${componentToHex(g)}${componentToHex(b)}`; +/* Selects the (hopefully) best color from Twitter's palette */ export const colorFromPalette = (palette: MediaPlaceholderColor[]) => { for (let i = 0; i < palette.length; i++) { const rgb = palette[i].rgb; diff --git a/src/pollHelper.ts b/src/helpers/pollTime.ts similarity index 94% rename from src/pollHelper.ts rename to src/helpers/pollTime.ts index 898a945..bda2ef4 100644 --- a/src/pollHelper.ts +++ b/src/helpers/pollTime.ts @@ -1,4 +1,6 @@ -import { Strings } from './strings'; +/* Helps create strings for polls! */ + +import { Strings } from '../strings'; export const calculateTimeLeft = (date: Date) => { const now = new Date(); diff --git a/src/quote.ts b/src/helpers/quote.ts similarity index 82% rename from src/quote.ts rename to src/helpers/quote.ts index cd1396a..f72a009 100644 --- a/src/quote.ts +++ b/src/helpers/quote.ts @@ -1,5 +1,6 @@ -import { Strings } from './strings'; +import { Strings } from '../strings'; +/* Helper for Quote Tweets */ export const handleQuote = (quote: APITweet): string | null => { console.log('Quoting status ', quote.id); diff --git a/src/translate.ts b/src/helpers/translate.ts similarity index 94% rename from src/translate.ts rename to src/helpers/translate.ts index 6df9983..fa8921c 100644 --- a/src/translate.ts +++ b/src/helpers/translate.ts @@ -1,5 +1,6 @@ -import { Constants } from './constants'; +import { Constants } from '../constants'; +/* Handles translating Tweets when asked! */ export const translateTweet = async ( tweet: TweetPartial, guestToken: string, diff --git a/src/utils.ts b/src/helpers/utils.ts similarity index 100% rename from src/utils.ts rename to src/helpers/utils.ts diff --git a/src/status.ts b/src/status.ts index 6330095..752ba17 100644 --- a/src/status.ts +++ b/src/status.ts @@ -1,8 +1,8 @@ import { Constants } from './constants'; -import { handleQuote } from './quote'; -import { sanitizeText } from './utils'; +import { handleQuote } from './helpers/quote'; +import { sanitizeText } from './helpers/utils'; import { Strings } from './strings'; -import { getAuthorText } from './author'; +import { getAuthorText } from './helpers/author'; import { statusAPI } from './api'; export const returnError = (error: string): StatusResponse => { @@ -17,18 +17,22 @@ export const returnError = (error: string): StatusResponse => { }; }; +/* Handler for Twitter statuses (Tweets). + Like Twitter, we use the terminologies interchangably. */ export const handleStatus = async ( status: string, mediaNumber?: number, userAgent?: string, flags?: InputFlags, language?: string + // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { console.log('Direct?', flags?.direct); const api = await statusAPI(status, language); const tweet = api?.tweet as APITweet; + /* Catch this request if it's an API response */ if (flags?.api) { return { response: new Response(JSON.stringify(api), { @@ -38,6 +42,7 @@ export const handleStatus = async ( }; } + /* If there was any errors fetching the Tweet, we'll return it */ switch (api.code) { case 401: return returnError(Strings.ERROR_PRIVATE); @@ -47,6 +52,7 @@ export const handleStatus = async ( return returnError(Strings.ERROR_API_FAIL); } + /* Catch direct media request (d.fxtwitter.com, or .mp4 / .jpg) */ if (flags?.direct && tweet.media) { let redirectUrl: string | null = null; if (tweet.media.videos) { @@ -61,12 +67,15 @@ export const handleStatus = async ( } } - /* Use quote media if there is no media */ + /* Use quote media if there is no media in this Tweet */ if (!tweet.media && tweet.quote?.media) { tweet.media = tweet.quote.media; tweet.twitter_card = tweet.quote.twitter_card; } + /* At this point, we know we're going to have to create a + regular embed because it's not an API or direct media request */ + let authorText = getAuthorText(tweet) || Strings.DEFAULT_AUTHOR_TEXT; const engagementText = authorText.replace(/ {4}/g, ' '); let siteName = Constants.BRANDING_NAME; @@ -80,12 +89,17 @@ export const handleStatus = async ( `` ]; + /* This little thing ensures if by some miracle a FixTweet embed is loaded in a browser, + it will gracefully redirect to the destination instead of just seeing a blank screen. + + Telegram is dumb and it just gets stuck if this is included, so we never include it for Telegram UAs. */ if (userAgent?.indexOf('Telegram') === -1) { headers.push( `` ); } + /* This Tweet has a translation attached to it, so we'll render it. */ if (tweet.translation) { const { translation } = tweet; @@ -102,7 +116,10 @@ export const handleStatus = async ( newText = `${translation.text}\n\n` + `${formatText}\n\n` + `${newText}`; } - /* Video renderer */ + /* This Tweet has a video to render. + + Twitter supports multiple videos in a Tweet now. But we have no mechanism to embed more than one. + You can still use /video/:number to get a specific video. Otherwise, it'll pick the first. */ if (tweet.media?.videos) { authorText = newText || ''; @@ -113,8 +130,13 @@ export const handleStatus = async ( const { videos } = tweet.media; const video = videos[(mediaNumber || 1) - 1]; - /* Multiplying by 0.5 is an ugly hack to fix Discord - disliking videos that are too large lol */ + /* This fix is specific to Discord not wanting to render videos that are too large, + or rendering low quality videos too small. + + Basically, our solution is to cut the dimensions in half if the video is too big (> 1080p), + or double them if it's too small. (<400p) + + We check both height and width so we can apply this to both horizontal and vertical videos equally*/ let sizeMultiplier = 1; @@ -125,6 +147,8 @@ export const handleStatus = async ( sizeMultiplier = 2; } + /* Like photos when picking a specific one (not using mosaic), + we'll put an indicator if there are more than one video */ if (videos.length > 1) { const videoCounter = Strings.VIDEO_COUNT.format({ number: String(videos.indexOf(video) + 1), @@ -143,9 +167,9 @@ export const handleStatus = async ( } } - headers.push(``); - + /* Push the raw video-related headers */ headers.push( + ``, ``, ``, ``, @@ -158,13 +182,16 @@ export const handleStatus = async ( ); } - /* Photo renderer */ + /* This Tweet has one or more photos to render */ if (tweet.media?.photos) { const { photos } = tweet.media; let photo = photos[(mediaNumber || 1) - 1]; + /* If there isn't a specified media number and we have a + mosaic response, we'll render it using mosaic */ if (typeof mediaNumber !== 'number' && tweet.media.mosaic) { photo = { + /* Telegram is dumb and doesn't support webp in opengraph embeds */ url: userAgent?.indexOf('Telegram') === -1 ? tweet.media.mosaic.formats.webp @@ -173,6 +200,8 @@ export const handleStatus = async ( height: tweet.media.mosaic.height, type: 'photo' }; + /* If mosaic isn't available or the link calls for a specific photo, + we'll indicate which photo it is out of the total */ } else if (photos.length > 1) { const photoCounter = Strings.PHOTO_COUNT.format({ number: String(photos.indexOf(photo) + 1), @@ -191,6 +220,7 @@ export const handleStatus = async ( } } + /* Push the raw photo-related headers */ headers.push( ``, ``, @@ -201,7 +231,7 @@ export const handleStatus = async ( ); } - /* External media renderer (i.e. YouTube) */ + /* We have external media available to us (i.e. YouTube videos) */ if (tweet.media?.external) { const { external } = tweet.media; authorText = newText || ''; @@ -217,32 +247,41 @@ export const handleStatus = async ( ); } - /* Poll renderer */ + /* This Tweet contains a poll, so we'll render it */ if (tweet.poll) { const { poll } = tweet; let barLength = 36; let str = ''; + /* Telegram Embeds are smaller, so we use a smaller bar to compensate */ if (userAgent?.indexOf('Telegram') !== -1) { barLength = 24; } + /* Render each poll choice */ tweet.poll.choices.forEach(choice => { - // render bar const bar = '█'.repeat((choice.percentage / 100) * barLength); // eslint-disable-next-line no-irregular-whitespace - str += `${bar}\n${choice.label}  (${choice.percentage}%) -`; + str += `${bar}\n${choice.label}  (${choice.percentage}%)\n`; }); + /* Finally, add the footer of the poll with # of votes and time left */ str += `\n${poll.total_votes} votes · ${poll.time_left_en}`; + /* And now we'll put the poll right after the Tweet text! */ newText += `\n\n${str}`; } + /* This Tweet quotes another Tweet, so we'll render the other Tweet where possible */ + if (api.tweet?.quote) { + const quoteText = handleQuote(api.tweet.quote); + newText += `\n${quoteText}`; + } + + /* If we have no media to display, instead we'll display the user profile picture in the embed */ if (!tweet.media?.video && !tweet.media?.photos) { headers.push( - // Use a slightly higher resolution image for profile pics + /* Use a slightly higher resolution image for profile pics */ ``, ``, @@ -265,6 +300,9 @@ export const handleStatus = async ( /* Special reply handling if authorText is not overriden */ if (tweet.replying_to && authorText === Strings.DEFAULT_AUTHOR_TEXT) { authorText = `↪ Replying to @${tweet.replying_to}`; + /* We'll assume it's a thread if it's a reply to themselves */ + } else if (tweet.replying_to === tweet.author.screen_name) { + authorText = `↪ A part @${tweet.author.screen_name}'s thread`; } /* The additional oembed is pulled by Discord to enable improved embeds. @@ -277,9 +315,10 @@ export const handleStatus = async ( )}" type="application/json+oembed" title="${tweet.author.name}">` ); - /* When dealing with a Tweet of unknown lang, fall back to en */ - const lang = tweet.lang === 'unk' ? 'en' : tweet.lang || 'en'; + /* When dealing with a Tweet of unknown lang, fall back to en */ + const lang = tweet.lang === null ? 'en' : tweet.lang || 'en'; + /* Finally, after all that work we return the response HTML! */ return { text: Strings.BASE_HTML.format({ lang: `lang="${lang}"`, diff --git a/src/strings.ts b/src/strings.ts index 55cd7b7..98bb5c6 100644 --- a/src/strings.ts +++ b/src/strings.ts @@ -4,10 +4,7 @@ declare global { } } -/* - Useful little function to format strings for us -*/ - +/* Useful little function to format strings for us */ String.prototype.format = function (options: { [find: string]: string }) { return this.replace(/{([^{}]+)}/g, (match: string, name: string) => { if (options[name] !== undefined) { @@ -17,6 +14,7 @@ String.prototype.format = function (options: { [find: string]: string }) { }); }; +/* Lots of strings! These are strings used in HTML or are shown to end users in embeds. */ export const Strings = { BASE_HTML: `