Merge pull request #781 from FixTweet/i18n

Implement internationalization (i18n)
This commit is contained in:
dangered wolf 2024-05-01 00:35:06 -04:00 committed by GitHub
commit 8fcf12075b
Signed by: DevComp
GPG key ID: B5690EEEBB952194
12 changed files with 594 additions and 833 deletions

70
i18n/resources.json Normal file
View file

@ -0,0 +1,70 @@
{
"en": {
"translation": {
"translatedFrom": "Translated from {language}",
"quotedFrom": "Quoting {name} (@{screen_name})",
"replyingTo": "Replying to @{screen_name}",
"threadPartHeader": "A part of @${screen_name}'s thread",
"ivAuthorActionReply": "<a href=\"{statusUrl}\">Reply</a> from <b>{authorName}</b> (<a href=\"{authorUrl}\">@{authorScreenName}</a>):",
"ivAuthorActionOriginal": "<a href=\"{statusUrl}\">Original</a> from <b>{authorName}</b> (<a href=\"{authorUrl}\">@{authorScreenName}</a>):",
"ivAuthorActionFollowUp": "<a href=\"{statusUrl}\">Follow-up</a> from <b>{authorName}</b> (<a href=\"{authorUrl}\">@{authorScreenName}</a>):",
"ivQuoteHeader": "<a href=\"{url}\">Quoting</a> {authorName} (<a href=\"{authorURL}\">@{authorHandle}</a>)",
"photoCount": "Photo {number} / {total}",
"videoCount": "Video {number} / {total}",
"mediaCount": "Media {number} / {total}",
"videoAltTextUnavailable": "{author}'s video. Alt text not available.",
"gifAltTextUnavailable": "{author}'s GIF. Alt text not available.",
"ivOriginalText": "Original text",
"ivViewOriginal": "View full thread",
"ivAboutAuthor": "About author",
"ivProfileFollowing": "{numFollowing, plural,\none {Following}\nother {Following}\n}",
"ivProfileFollowers": "{numFollowers, plural,\none {Follower}\nother {Followers}\n}",
"ivProfileStatuses": "{numStatuses, plural,\none {Post}\nother {Posts}\n}",
"ivProfilePictureAlt": "{author}'s profile picture",
"ivFallbackText": "If you can see this, your browser is doing something weird with your user agent.",
"ivInternetArchiveText": "{brandingName} archive",
"pollFinalResults": "Final results",
"pollVotes": "{voteCount, plural,\none {# vote}\nother {# votes}\n} · {timeLeft}",
"ivPollChoice": "{voteCount, plural,\none {# vote}\nother {# votes}\n}, {percentage}%",
"ivCommunityNoteHeader": "Readers added context they thought people might want to know",
"gifIndicator": "GIF - {brandingName}",
"language_af": "Afrikaans",
"language_ar": "Arabic",
"language_ca": "Catalan",
"language_cs": "Czech",
"language_da": "Danish",
"language_de": "German",
"language_en": "English",
"language_el": "Greek",
"language_es": "Spanish",
"language_fi": "Finnish",
"language_fr": "French",
"language_he": "Hebrew",
"language_hu": "Hungarian",
"language_it": "Italian",
"language_ja": "Japanese",
"language_ko": "Korean",
"language_nl": "Dutch",
"language_no": "Norwegian",
"language_pl": "Polish",
"language_pt": "Portuguese",
"language_ro": "Romanian",
"language_ru": "Russian",
"language_sr": "Serbian",
"language_sv": "Swedish",
"language_tr": "Turkish",
"language_uk": "Ukrainian",
"language_vi": "Vietnamese",
"language_zh-CN": "Chinese",
"language_zh-TW": "Chinese"
}
}
}

1110
package-lock.json generated

File diff suppressed because it is too large Load diff

View file

@ -39,6 +39,8 @@
},
"dependencies": {
"@hono/sentry": "^1.0.1",
"i18next": "^23.8.2",
"i18next-icu": "^2.3.0",
"hono": "^4.2.9"
}
}

View file

@ -15,7 +15,7 @@ export const Constants = {
REDIRECT_URL: REDIRECT_URL,
RELEASE_NAME: RELEASE_NAME,
GIF_TRANSCODE_DOMAIN: GIF_TRANSCODE_DOMAIN,
API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`,
API_DOCS_URL: `https://github.com/FixTweet/FxTwitter/wiki/API-Home`,
TWITTER_ROOT: 'https://twitter.com',
TWITTER_GLOBAL_NAME_ROOT: 'twitter.com',
TWITTER_API_ROOT: 'https://api.twitter.com',

View file

@ -1,4 +1,7 @@
import { Context } from 'hono';
import { StatusCode } from 'hono/utils/http-status';
import i18next from 'i18next';
import icu from "i18next-icu";
import { Constants } from '../constants';
import { handleQuote } from '../helpers/quote';
import { formatNumber, sanitizeText, truncateWithEllipsis } from '../helpers/utils';
@ -9,7 +12,7 @@ import { renderVideo } from '../render/video';
import { renderInstantView } from '../render/instantview';
import { constructTwitterThread } from '../providers/twitter/conversation';
import { Experiment, experimentCheck } from '../experiments';
import { StatusCode } from 'hono/utils/http-status';
import translationResources from '../../i18n/resources.json';
export const returnError = (c: Context, error: string): Response => {
return c.html(
@ -127,6 +130,13 @@ export const handleStatus = async (
let overrideMedia: APIMedia | undefined;
await i18next.use(icu).init({
lng: language ?? status.lang ?? 'en',
debug: true,
resources: translationResources,
fallbackLng: 'en'
});
// Check if mediaNumber exists, and if that media exists in status.media.all. If it does, we'll store overrideMedia variable
if (mediaNumber && status.media && status.media.all && status.media.all[mediaNumber - 1]) {
overrideMedia = status.media.all[mediaNumber - 1];
@ -200,7 +210,8 @@ export const handleStatus = async (
status: status,
thread: thread,
text: newText,
flags: flags
flags: flags,
targetLanguage: language ?? status.lang ?? 'en'
});
headers.push(...instructions.addHeaders);
if (instructions.authorText) {
@ -208,7 +219,7 @@ export const handleStatus = async (
}
ivbody = instructions.text || '';
} catch (e) {
console.log('Error rendering Instant View', e);
console.log('Error rendering Instant View', e, (e as Error)?.stack);
useIV = false;
}
}
@ -219,15 +230,11 @@ export const handleStatus = async (
if (status.translation) {
const { translation } = status;
const formatText =
language === 'en'
? Strings.TRANSLATE_TEXT.format({
language: translation.source_lang_en
})
: Strings.TRANSLATE_TEXT_INTL.format({
source: translation.source_lang.toUpperCase(),
destination: translation.target_lang.toUpperCase()
});
const formatText = `📑 {translation}`.format({
translation: i18next.t('translatedFrom').format({
language: i18next.t(`language_${translation.source_lang}`)
})
});
newText = `${formatText}\n\n` + `${translation.text}\n\n`;
}
@ -381,7 +388,10 @@ export const handleStatus = async (
});
/* Finally, add the footer of the poll with # of votes and time left */
str += `\n${formatNumber(poll.total_votes)} votes · ${poll.time_left_en}`;
str += '\n'; /* TODO: Localize time left */
str += i18next
.t('pollVotes')
.format({ voteCount: formatNumber(poll.total_votes), timeLeft: poll.time_left_en });
/* Check if the poll is ongoing and apply low TTL cache control.
Yes, checking if this is a string is a hacky way to do this, but
@ -455,13 +465,13 @@ export const handleStatus = async (
/* Special reply handling if authorText is not overriden */
if (status.replying_to && authorText === Strings.DEFAULT_AUTHOR_TEXT) {
authorText = `Replying to @${status.replying_to.screen_name}`;
authorText = `${i18next.t('replyingTo').format({ screen_name: status.replying_to.screen_name })}`;
/* We'll assume it's a thread if it's a reply to themselves */
} else if (
status.replying_to?.screen_name === status.author.screen_name &&
authorText === Strings.DEFAULT_AUTHOR_TEXT
) {
authorText = `A part of @${status.author.screen_name}'s thread`;
authorText = `${i18next.t('threadPartHeader').format({ screen_name: status.author.screen_name })}`;
}
if (!flags.gallery) {
@ -479,7 +489,7 @@ export const handleStatus = async (
const mediaType = overrideMedia ?? status.media.videos?.[0]?.type;
if (mediaType === 'gif') {
provider = `GIF - ${Constants.BRANDING_NAME}`;
provider = i18next.t('gifIndicator', { brandingName: Constants.BRANDING_NAME });
} else if (
status.embed_card === 'player' &&
providerEngagementText !== Strings.DEFAULT_AUTHOR_TEXT

View file

@ -12,6 +12,7 @@ export const calculateTimeLeft = (date: Date) => {
return { days, hours, minutes, seconds };
};
/* TODO: Refactor to support pluralization of other languages */
export const calculateTimeLeftString = (date: Date) => {
const { days, hours, minutes, seconds } = calculateTimeLeft(date);
const daysString =

View file

@ -1,11 +1,11 @@
import { Strings } from '../strings';
import i18next from 'i18next';
/* Helper for Quote Tweets */
export const handleQuote = (quote: APIStatus): string | null => {
console.log('Quoting status ', quote.id);
let str = `\n`;
str += Strings.QUOTE_TEXT.format({
str += i18next.t('quotedFrom').format({
name: quote.author?.name || '',
screen_name: quote.author?.screen_name || ''
});

View file

@ -150,8 +150,8 @@ export const buildAPITwitterStatus = async (
apiStatus.replying_to_status = status.legacy?.in_reply_to_status_id_str || null;
} else if (status.legacy.in_reply_to_screen_name) {
apiStatus.replying_to = {
screen_name: status.legacy.in_reply_to_screen_name || null,
post: status.legacy.in_reply_to_status_id_str || null
screen_name: status.legacy.in_reply_to_screen_name,
post: status.legacy.in_reply_to_status_id_str
};
} else {
apiStatus.replying_to = null;

View file

@ -1,8 +1,8 @@
/* eslint-disable no-irregular-whitespace */
import i18next from 'i18next';
import { Constants } from '../constants';
import { getSocialTextIV } from '../helpers/socialproof';
import { sanitizeText } from '../helpers/utils';
import { Strings } from '../strings';
enum AuthorActionType {
Reply = 'Reply',
@ -10,7 +10,7 @@ enum AuthorActionType {
FollowUp = 'FollowUp'
}
const populateUserLinks = (status: APIStatus, text: string): string => {
const populateUserLinks = (text: string): string => {
/* TODO: Maybe we can add username splices to our API so only genuinely valid users are linked? */
text.match(/@(\w{1,15})/g)?.forEach(match => {
const username = match.replace('@', '');
@ -22,7 +22,7 @@ const populateUserLinks = (status: APIStatus, text: string): string => {
return text;
};
const generateStatusMedia = (status: APIStatus, author: APIUser): string => {
const generateStatusMedia = (status: APIStatus): string => {
let media = '';
if (status.media?.all?.length) {
status.media.all.forEach(mediaItem => {
@ -36,10 +36,10 @@ const generateStatusMedia = (status: APIStatus, author: APIUser): string => {
});
break;
case 'video':
media += `<video src="${mediaItem.url}" alt="${author.name}'s video. Alt text not available."/>`;
media += `<video src="${mediaItem.url}" alt="${i18next.t('videoAltTextUnavailable').format({ author: status.author.name })}"/>`;
break;
case 'gif':
media += `<video src="${mediaItem.url}" alt="${author.name}'s gif. Alt text not available."/>`;
media += `<video src="${mediaItem.url}" alt="${i18next.t('gifAltTextUnavailable').format({ author: status.author.name })}"/>`;
break;
}
});
@ -56,11 +56,16 @@ const generateStatusMedia = (status: APIStatus, author: APIUser): string => {
// return `${hh}:${min} - ${yyyy}/${mm}/${dd}`;
// }
const formatDate = (date: Date): string => {
const yyyy = date.getFullYear();
const mm = String(date.getMonth() + 1).padStart(2, '0'); // Months are 0-indexed
const dd = String(date.getDate()).padStart(2, '0');
return `${yyyy}/${mm}/${dd}`;
const formatDate = (date: Date, language: string): string => {
if (language.startsWith('en')) {
language = 'en-CA'; // Use ISO dates for English to avoid problems with mm/dd vs. dd/mm
}
const formatter = new Intl.DateTimeFormat(language, {
year: 'numeric',
month: '2-digit',
day: '2-digit'
});
return formatter.format(date);
};
const htmlifyLinks = (input: string): string => {
@ -95,32 +100,27 @@ function getTranslatedText(status: APITwitterStatus, isQuote = false): string |
let text = paragraphify(sanitizeText(status.translation?.text), isQuote);
text = htmlifyLinks(text);
text = htmlifyHashtags(text);
text = populateUserLinks(status, text);
text = populateUserLinks(text);
const formatText =
status.translation.target_lang === 'en'
? Strings.TRANSLATE_TEXT.format({
language: status.translation.source_lang_en
})
: Strings.TRANSLATE_TEXT_INTL.format({
source: status.translation.source_lang.toUpperCase(),
destination: status.translation.target_lang.toUpperCase()
});
const formatText = `📑 {translation}`.format({
translation: i18next.t('translatedFrom').format({
language: i18next.t(`language_${status.translation.source_lang}`)
})
});
return `<h4>${formatText}</h4>${text}<h4>Original</h4>`;
return `<h4>${formatText}</h4>${text}<h4>${i18next.t('ivOriginalText')}</h4>`;
}
const notApplicableComment = '<!-- N/A -->';
// 1100 -> 1.1K, 1100000 -> 1.1M
const truncateSocialCount = (count: number): string => {
if (count >= 1000000) {
return `${(count / 1000000).toFixed(1)}M`;
} else if (count >= 1000) {
return `${(count / 1000).toFixed(1)}K`;
} else {
return String(count);
}
const truncateSocialCount = (count: number, locale = 'en-US') => {
const formatter = new Intl.NumberFormat(locale, {
notation: 'compact',
compactDisplay: 'short',
maximumFractionDigits: 1
});
return formatter.format(count);
};
const generateInlineAuthorHeader = (
@ -128,16 +128,28 @@ const generateInlineAuthorHeader = (
author: APIUser,
authorActionType: AuthorActionType | null
): string => {
return `<h4><i><a href="${status.url}">{AuthorAction}</a> from <b>${author.name}</b> (<a href="${author.url}">@${author.screen_name}</a>):</i></h4>`.format(
{
AuthorAction:
authorActionType === AuthorActionType.Reply
? 'Reply'
: authorActionType === AuthorActionType.Original
? 'Original'
: 'Follow-up'
}
);
if (authorActionType === AuthorActionType.Original) {
return `<h4><i>${i18next.t('ivAuthorActionOriginal', {
statusUrl: status.url,
authorName: author.name,
authorUrl: author.url,
authorScreenName: author.screen_name
})}</i></h4>`;
} else if (authorActionType === AuthorActionType.FollowUp) {
return `<h4><i>${i18next.t('ivAuthorActionFollowUp', {
statusUrl: status.url,
authorName: author.name,
authorUrl: author.url,
authorScreenName: author.screen_name
})}</i></h4>`;
}
// Reply / unknown
return `<h4><i>${i18next.t('ivAuthorActionReply', {
statusUrl: status.url,
authorName: author.name,
authorUrl: author.url,
authorScreenName: author.screen_name
})}</i></h4>`;
};
const wrapForeignLinks = (url: string) => {
@ -158,11 +170,16 @@ const wrapForeignLinks = (url: string) => {
: url;
};
const generateStatusFooter = (status: APIStatus, isQuote = false, author: APIUser): string => {
const generateStatusFooter = (
status: APIStatus,
isQuote = false,
author: APIUser,
language: string
): string => {
let description = author.description;
description = htmlifyLinks(description);
description = htmlifyHashtags(description);
description = populateUserLinks(status, description);
description = populateUserLinks(description);
return `
<p>{socialText}</p>
@ -171,31 +188,31 @@ const generateStatusFooter = (status: APIStatus, isQuote = false, author: APIUse
{aboutSection}
`.format({
socialText: getSocialTextIV(status as APITwitterStatus) || '',
viewOriginal: !isQuote ? `<a href="${status.url}">View full thread</a>` : notApplicableComment,
viewOriginal: !isQuote
? `<a href="${status.url}">${i18next.t('ivViewOriginal')}</a>`
: notApplicableComment,
aboutSection: isQuote
? ''
: `<h2>About author</h2>
: `<h2>${i18next.t('ivAboutAuthor')}</h2>
{pfp}
<h2>${author.name}</h2>
<p><a href="${author.url}">@${author.screen_name}</a></p>
<p><b>${description}</b></p>
<p>{location} {website} {joined}</p>
<p>
{following} <b>Following</b>
{followers} <b>Followers</b>
{statuses} <b>Posts</b>
{following} <b>${i18next.t('ivProfileFollowing', { numFollowing: author.following })}</b>
{followers} <b>${i18next.t('ivProfileFollowers', { numFollowers: author.followers })}</b>
{statuses} <b>${i18next.t('ivProfileStatuses', { numStatuses: author.statuses })}</b>
</p>`.format({
pfp: `<img src="${author.avatar_url?.replace('_200x200', '_400x400')}" alt="${
author.name
}'s profile picture" />`,
pfp: `<img src="${author.avatar_url?.replace('_200x200', '_400x400')}" alt="${i18next.t('ivProfilePictureAlt', { author: author.name })}" />`,
location: author.location ? `📌 ${author.location}` : '',
website: author.website
? `🔗 <a rel="nofollow" href="${wrapForeignLinks(author.website.url)}">${author.website.display_url}</a>`
: '',
joined: author.joined ? `📆 ${formatDate(new Date(author.joined))}` : '',
following: truncateSocialCount(author.following),
followers: truncateSocialCount(author.followers),
statuses: truncateSocialCount(author.statuses)
joined: author.joined ? `📆 ${formatDate(new Date(author.joined), language)}` : '',
following: truncateSocialCount(author.following, language),
followers: truncateSocialCount(author.followers, language),
statuses: truncateSocialCount(author.statuses, language)
})
});
};
@ -209,10 +226,10 @@ const generatePoll = (poll: APIPoll, language: string): string => {
poll.choices.forEach(choice => {
const bar = '█'.repeat((choice.percentage / 100) * barLength);
// eslint-disable-next-line no-irregular-whitespace
str += `${bar}<br>${choice.label}<br>${intlFormat.format(choice.count)} votes, ${intlFormat.format(choice.percentage)}%<br>`;
str += `${bar}<br>${choice.label}<br>${i18next.t('ivPollChoice', { voteCount: intlFormat.format(choice.count), percentage: intlFormat.format(choice.percentage) })}<br>`;
});
/* Finally, add the footer of the poll with # of votes and time left */
str += `<br>${intlFormat.format(poll.total_votes)} votes · ${poll.time_left_en}`;
str += `<br>${i18next.t('pollVotes', { voteCount: intlFormat.format(poll.total_votes), timeLeft: poll.time_left_en })}`;
return str;
};
@ -247,7 +264,7 @@ const generateCommunityNote = (status: APITwitterStatus): string => {
// Add the remaining text after the last link
result = `<table>
<thead>
<th><b>Readers added context they thought people might want to know</b></th>
<th><b>${i18next.t('ivCommunityNoteHeader')}</b></th>
</thead>
<tbody>
<th>${result.replace(/\n/g, '\n<br>')}</th>
@ -262,20 +279,21 @@ const generateCommunityNote = (status: APITwitterStatus): string => {
const generateStatus = (
status: APIStatus,
author: APIUser,
language: string,
isQuote = false,
authorActionType: AuthorActionType | null
): string => {
let text = paragraphify(sanitizeText(status.text), isQuote);
text = htmlifyLinks(text);
text = htmlifyHashtags(text);
text = populateUserLinks(status, text);
text = populateUserLinks(text);
const translatedText = getTranslatedText(status as APITwitterStatus, isQuote);
return `<!-- Telegram Instant View -->
{quoteHeader}
<!-- Embed media -->
${generateStatusMedia(status, author)}
${generateStatusMedia(status)}
<!-- Translated text (if applicable) -->
${translatedText ? translatedText : notApplicableComment}
<!-- Inline author (if applicable) -->
@ -287,12 +305,20 @@ const generateStatus = (
<!-- Embed poll -->
${status.poll ? generatePoll(status.poll, status.lang ?? 'en') : notApplicableComment}
<!-- Embedded quote status -->
${!isQuote && status.quote ? generateStatus(status.quote, author, true, null) : notApplicableComment}
`.format({
quoteHeader: isQuote
? `<h4><a href="${status.url}">Quoting</a> ${status.author.name} (<a href="${Constants.TWITTER_ROOT}/${status.author.screen_name}">@${status.author.screen_name}</a>)</h4>`
: ''
});
${!isQuote && status.quote ? generateStatus(status.quote, author, language, true, null) : notApplicableComment}`.format(
{
quoteHeader: isQuote
? '<h4>' +
i18next.t('ivQuoteHeader').format({
url: status.url,
authorName: status.author.name,
authorHandle: status.author.screen_name,
authorURL: `${Constants.TWITTER_ROOT}/${status.author.screen_name}`
}) +
'</h4>'
: ''
}
);
};
export const renderInstantView = (properties: RenderProperties): ResponseInstructions => {
@ -331,12 +357,12 @@ export const renderInstantView = (properties: RenderProperties): ResponseInstruc
</section>
<section class="section--first">${
flags?.archive
? `${Constants.BRANDING_NAME} archive`
: 'If you can see this, your browser is doing something weird with your user agent.'
} <a href="${status.url}">View full thread</a>
? i18next.t('ivInternetArchiveText').format({ brandingName: Constants.BRANDING_NAME })
: i18next.t('ivFallbackText')
} <a href="${status.url}">${i18next.t('ivViewOriginal')}</a>
</section>
<article>
<sub><a href="${status.url}">View full thread</a></sub>
<sub><a href="${status.url}">${i18next.t('ivViewOriginal')}</a></sub>
<h1>${status.author.name} (@${status.author.screen_name})</h1>
${useThread
@ -376,11 +402,17 @@ export const renderInstantView = (properties: RenderProperties): ResponseInstruc
previousThreadPieceAuthor = status.author?.id;
return generateStatus(status, status.author ?? thread?.author, false, authorAction);
return generateStatus(
status,
status.author ?? thread?.author,
properties?.targetLanguage ?? 'en',
false,
authorAction
);
})
.join('')}
${generateStatusFooter(status, false, thread?.author ?? status.author)}
<br>${`<a href="${status.url}">View full thread</a>`}
${generateStatusFooter(status, false, thread?.author ?? status.author, properties?.targetLanguage ?? 'en')}
<br>${`<a href="${status.url}">${i18next.t('ivViewOriginal')}</a>`}
</article>`;
return instructions;

View file

@ -1,3 +1,4 @@
import i18next from 'i18next';
import { Constants } from '../constants';
import { Strings } from '../strings';
@ -13,7 +14,9 @@ export const renderPhoto = (
const all = status.media?.all as APIMedia[];
const baseString =
all.length === status.media?.photos?.length ? Strings.PHOTO_COUNT : Strings.MEDIA_COUNT;
all.length === status.media?.photos?.length
? i18next.t('photoCount')
: i18next.t('mediaCount');
const photoCounter = baseString.format({
number: String(all.indexOf(photo) + 1),

View file

@ -1,7 +1,7 @@
import i18next from 'i18next';
import { Constants } from '../constants';
import { Experiment, experimentCheck } from '../experiments';
import { handleQuote } from '../helpers/quote';
import { Strings } from '../strings';
export const renderVideo = (
properties: RenderProperties,
@ -33,7 +33,9 @@ export const renderVideo = (
we'll put an indicator if there are more than one video */
if (all && all.length > 1 && (userAgent?.indexOf('Telegram') ?? 0) > -1) {
const baseString =
all.length === status.media?.videos?.length ? Strings.VIDEO_COUNT : Strings.MEDIA_COUNT;
all.length === status.media?.videos?.length
? i18next.t('videoCount')
: i18next.t('mediaCount');
const videoCounter = baseString.format({
number: String(all.indexOf(video) + 1),
total: String(all.length)

View file

@ -38,6 +38,7 @@ interface RenderProperties {
userAgent?: string;
text?: string;
flags?: InputFlags;
targetLanguage?: string;
}
interface TweetAPIResponse {
@ -141,8 +142,8 @@ interface APIStatus {
possibly_sensitive: boolean;
replying_to: {
screen_name: string | null;
post: string | null;
screen_name: string;
post: string;
} | null;
source: string | null;