From 639fd7851366a68e5417258c9ebc95acb6c65c00 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 17 Oct 2023 04:30:35 -0400 Subject: [PATCH 01/16] Bump fakeChromeVersion --- src/helpers/useragent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/useragent.ts b/src/helpers/useragent.ts index 512d9bb..d09d536 100644 --- a/src/helpers/useragent.ts +++ b/src/helpers/useragent.ts @@ -1,6 +1,6 @@ /* We keep this value up-to-date for making our requests to Twitter as indistinguishable from normal user traffic as possible. */ -const fakeChromeVersion = 116; +const fakeChromeVersion = 118; const platformWindows = 'Windows NT 10.0; Win64; x64'; const platformMac = 'Macintosh; Intel Mac OS X 10_15_7'; const platformLinux = 'X11; Linux x86_64'; From 63f1e9d9ef79d06eb905a23d32c12928e9f101a3 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 17 Oct 2023 21:53:35 -0400 Subject: [PATCH 02/16] Update package-lock.json --- package-lock.json | 172 +++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ae70e6..007aff5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -716,9 +716,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20231010.0.tgz", - "integrity": "sha512-LM9ePAh88EGoQkYisAfdLMEDzcaMinRer0mY11GOiN4A9ZU+6APRVvhh5JBRzI0F6Dkb8nHtrzhisioWCRaY1w==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20231016.0.tgz", + "integrity": "sha512-rPAnF8Q25+eHEsAopihWeftPW/P0QapY9d7qaUmtOXztWdd6YPQ7JuiWVj4Nvjphge1BleehxAbo4I3Z4L2H1g==", "cpu": [ "x64" ], @@ -732,9 +732,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20231010.0.tgz", - "integrity": "sha512-Vr7Z1O+vJRCnVeWaF0YSv0EMHiMRY7yYCxr7O509FzvJAXsZuXZ7DYC5TAD7a8HSeeqsxFTAbF9jg0y9A2wKVw==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20231016.0.tgz", + "integrity": "sha512-MvydDdiLXt+jy57vrVZ2lU6EQwCdpieyZoN8uBXSWzfG3zR/6dxU1+okvPQPlHN0jtlufqPeHrpJyAqqgLHUKA==", "cpu": [ "arm64" ], @@ -748,9 +748,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20231010.0.tgz", - "integrity": "sha512-l9oDVPVhPEOHr1JpcGnLSsIf1h8sZnvcIC2Tl1zt+3p/KGFyGqGyAZJMLUoMJ54Q07oRE1x3KAu+JcWWEvdxpg==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20231016.0.tgz", + "integrity": "sha512-y6Sj37yTzM8QbAghG9LRqoSBrsREnQz8NkcmpjSxeK6KMc2g0L5A/OemCdugNlIiv+zRv9BYX1aosaoxY5JbeQ==", "cpu": [ "x64" ], @@ -764,9 +764,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20231010.0.tgz", - "integrity": "sha512-NBmYsJu+ns2W8WHcDnglfqLV5O3FP7lXpoTSTvpM64mhexmemdMlOJX5gpRuarTula3fA+GzEehinUojwM9/1g==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20231016.0.tgz", + "integrity": "sha512-LqMIRUHD1YeRg2TPIfIQEhapSKMFSq561RypvJoXZvTwSbaROxGdW6Ku+PvButqTkEvuAtfzN/kGje7fvfQMHg==", "cpu": [ "arm64" ], @@ -780,9 +780,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20231010.0.tgz", - "integrity": "sha512-jWiG71Rvuh4FYdEpOP1+BAygdguTlMYYy+v5d4ZOjxDkl+V8aR86EEtDQrv/QLUJFbpcoEX25SxXnN5UMKtjhQ==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20231016.0.tgz", + "integrity": "sha512-96ojBwIHyiUAbsWlzBqo9P/cvH8xUh8SuBboFXtwAeXcJ6/urwKN2AqPa/QzOGUTCdsurWYiieARHT5WWWPhKw==", "cpu": [ "x64" ], @@ -796,9 +796,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "4.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20231010.0.tgz", - "integrity": "sha512-Jscqg52ScJMTMRHHVeoUvXYYiXPT4V59NKaClQVju9B6oPAzuePFYHwi5PLQ5Yr+xEbJ6+bHuq1aYV/69z6L0w==", + "version": "4.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20231016.0.tgz", + "integrity": "sha512-eGB0cRVyoJpeyGJx2re5sbd9R316a61sY73xwnqm4cwGpb+OxCK2gc651RxGiN7H4w6LY1RpysUgeGLmj5B3+g==", "dev": true }, "node_modules/@esbuild-plugins/node-globals-polyfill": { @@ -1735,9 +1735,9 @@ "dev": true }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.19", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", - "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz", + "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==", "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -2038,14 +2038,14 @@ } }, "node_modules/@sentry-internal/tracing": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.0.tgz", - "integrity": "sha512-JK6IRGgdtZjswGfaGIHNWIThffhOHzVIIaGmglui+VFIzOsOqePjoxaDV0MEvzafxXZD7eWqGE5RGuZ0n6HFVg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.1.tgz", + "integrity": "sha512-nNaiZreQxCitG2PzYPaC7XtyA9OMsETGYMKAtiK4p62/uTmeYbsBva9BoNx1XeiHRwbrVQYRMKQ9nV5e2jS4/A==", "dev": true, "dependencies": { - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2092,13 +2092,13 @@ } }, "node_modules/@sentry/core": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.0.tgz", - "integrity": "sha512-83NRuqn7nDZkSVBN5yJQqcpXDG4yMYiB7TkYUKrGTzBpRy6KUOrkCdybuKk0oraTIGiGSe5WEwCFySiNgR9FzA==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.1.tgz", + "integrity": "sha512-LvEhOSfdIvwkr+PdlrT/aA/iOLhkXrSkvjqAQyogE4ddCWeYfS0NoirxNt1EaxMBAWKhYZRqzkA7WA4LDLbzlA==", "dev": true, "dependencies": { - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2120,14 +2120,14 @@ } }, "node_modules/@sentry/integrations": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.74.0.tgz", - "integrity": "sha512-O4UyxiV5wzXSDnEd9Z/SIt/5M12URWNtIJPPJjowlllzw8X9e3zBcnXmjMOLZ+mZWjQmRDjOoz3lPPQ17f7fvw==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.74.1.tgz", + "integrity": "sha512-Q7chPehHpHB4WOQ1J/X6NiN2ptiqJMmxtL+6wHumzIAyrjup3c9XekR83qEs8zpqYJAlb/4MUlwd9fPbkhGXnQ==", "dev": true, "dependencies": { - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "localforage": "^1.8.1", "tslib": "^2.4.1 || ^1.9.3" }, @@ -2136,15 +2136,15 @@ } }, "node_modules/@sentry/node": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.0.tgz", - "integrity": "sha512-uBmW2/z0cz/WFIG74ZF7lSipO0XNzMf9yrdqnZXnGDYsUZE4I4QiqDN0hNi6fkTgf9MYRC8uFem2OkAvyPJ74Q==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.1.tgz", + "integrity": "sha512-aMUQ2LFZF64FBr+cgjAqjT4OkpYBIC9lyWI8QqjEHqNho5+LGu18/iVrJPD4fgs4UhGdCuAiQjpC36MbmnIDZA==", "dev": true, "dependencies": { - "@sentry-internal/tracing": "7.74.0", - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry-internal/tracing": "7.74.1", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "cookie": "^0.5.0", "https-proxy-agent": "^5.0.0", "lru_map": "^0.3.3", @@ -2155,21 +2155,21 @@ } }, "node_modules/@sentry/types": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.0.tgz", - "integrity": "sha512-rI5eIRbUycWjn6s6o3yAjjWtIvYSxZDdnKv5je2EZINfLKcMPj1dkl6wQd2F4y7gLfD/N6Y0wZYIXC3DUdJQQg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.1.tgz", + "integrity": "sha512-2jIuPc+YKvXqZETwr2E8VYnsH1zsSUR/wkIvg1uTVeVNyoowJv+YsOtCdeGyL2AwiotUBSPKu7O1Lz0kq5rMOQ==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/@sentry/utils": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.0.tgz", - "integrity": "sha512-k3np8nuTPtx5KDODPtULfFln4UXdE56MZCcF19Jv6Ljxf+YN/Ady1+0Oi3e0XoSvFpWNyWnglauT7M65qCE6kg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.1.tgz", + "integrity": "sha512-qUsqufuHYcy5gFhLZslLxA5kcEOkkODITXW3c7D+x+8iP/AJqa8v8CeUCVNS7RetHCuIeWAbbTClC4c411EwQg==", "dev": true, "dependencies": { - "@sentry/types": "7.74.0", + "@sentry/types": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2201,9 +2201,9 @@ } }, "node_modules/@types/babel__core": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.2.tgz", - "integrity": "sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA==", + "version": "7.20.3", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.3.tgz", + "integrity": "sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA==", "dev": true, "dependencies": { "@babel/parser": "^7.20.7", @@ -2214,18 +2214,18 @@ } }, "node_modules/@types/babel__generator": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.5.tgz", - "integrity": "sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.6.tgz", + "integrity": "sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w==", "dev": true, "dependencies": { "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__template": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.2.tgz", - "integrity": "sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.3.tgz", + "integrity": "sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ==", "dev": true, "dependencies": { "@babel/parser": "^7.1.0", @@ -2233,18 +2233,18 @@ } }, "node_modules/@types/babel__traverse": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.2.tgz", - "integrity": "sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw==", + "version": "7.20.3", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.3.tgz", + "integrity": "sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw==", "dev": true, "dependencies": { "@babel/types": "^7.20.7" } }, "node_modules/@types/better-sqlite3": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.5.tgz", - "integrity": "sha512-H3ZUx89KiPhYa9nalUXVVStSUFHuzYxt4yoazufpTTYW9rVUCzhh02V8CH2C8nE4libnK0UgFq5DFIe0DOhqow==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.6.tgz", + "integrity": "sha512-nuFAptzt0hZYBvyLzKQCbuCCK+RN9PHH4ezar5EJLIg2qpVhwQ/uLvLO/K8A9O7N8DafawgFupiyXQSs0U48Ng==", "dev": true, "dependencies": { "@types/node": "*" @@ -3494,9 +3494,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.554", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.554.tgz", - "integrity": "sha512-Q0umzPJjfBrrj8unkONTgbKQXzXRrH7sVV7D9ea2yBV3Oaogz991yhbpfvo2LMNkJItmruXTEzVpP9cp7vaIiQ==", + "version": "1.4.557", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.557.tgz", + "integrity": "sha512-6x0zsxyMXpnMJnHrondrD3SuAeKcwij9S+83j2qHAQPXbGTDDfgImzzwgGlzrIcXbHQ42tkG4qA6U860cImNhw==", "dev": true }, "node_modules/emittery": { @@ -5418,9 +5418,9 @@ } }, "node_modules/miniflare": { - "version": "3.20231010.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20231010.0.tgz", - "integrity": "sha512-VETY+/OhJ1RN+yrFpPUqBZysb2R8wXvyx3vzaRZS2qO1aGNKeGASa/vxCvNcBF+gt8UdbWMOalSXX8zY0IgWZA==", + "version": "3.20231016.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20231016.0.tgz", + "integrity": "sha512-AmlqI89zsnBJfC+nKKZdCB/fuu0q/br24Kqt9NZwcT6yJEpO5NytNKfjl6nJROHROwuJSRQR1T3yopCtG1/0DA==", "dev": true, "dependencies": { "acorn": "^8.8.0", @@ -5431,7 +5431,7 @@ "source-map-support": "0.5.21", "stoppable": "^1.1.0", "undici": "^5.22.1", - "workerd": "1.20231010.0", + "workerd": "1.20231016.0", "ws": "^8.11.0", "youch": "^3.2.2", "zod": "^3.20.6" @@ -7300,9 +7300,9 @@ } }, "node_modules/workerd": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20231010.0.tgz", - "integrity": "sha512-ghxfBU8fBSBDa8fCBPfzWivYsWpewYftgy70N308C+acQ5AaKNM1QTdkQNm9YWeC5Jpl1YvBX04ojt7lCc3juw==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20231016.0.tgz", + "integrity": "sha512-v2GDb5XitSqgub/xm7EWHVAlAK4snxQu3itdMQxXstGtUG9hl79fQbXS/8fNFbmms2R2bAxUwSv47q8k5T5Erw==", "dev": true, "hasInstallScript": true, "bin": { @@ -7312,17 +7312,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20231010.0", - "@cloudflare/workerd-darwin-arm64": "1.20231010.0", - "@cloudflare/workerd-linux-64": "1.20231010.0", - "@cloudflare/workerd-linux-arm64": "1.20231010.0", - "@cloudflare/workerd-windows-64": "1.20231010.0" + "@cloudflare/workerd-darwin-64": "1.20231016.0", + "@cloudflare/workerd-darwin-arm64": "1.20231016.0", + "@cloudflare/workerd-linux-64": "1.20231016.0", + "@cloudflare/workerd-linux-arm64": "1.20231016.0", + "@cloudflare/workerd-windows-64": "1.20231016.0" } }, "node_modules/wrangler": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.13.1.tgz", - "integrity": "sha512-CY73h4lfPx/3CmkC/tPj66DRRZ9Y42sMcHys6B6tjCILUo950IeOvnsj759el3/ewFLY4kG4jCrrrikan6TE+Q==", + "version": "3.13.2", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.13.2.tgz", + "integrity": "sha512-Z/ZrAL2mJc7E4ialOV9c3wGry0qagp9DfPl5XVd67vtlFPeTSR+1pemtZ5+qI2BXi59kP3OGHBKrrIyG0d9csg==", "dev": true, "dependencies": { "@cloudflare/kv-asset-handler": "^0.2.0", @@ -7331,7 +7331,7 @@ "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", "esbuild": "0.17.19", - "miniflare": "3.20231010.0", + "miniflare": "3.20231016.0", "nanoid": "^3.3.3", "path-to-regexp": "^6.2.0", "selfsigned": "^2.0.1", From 0a616e973178b57be751d4fbb9867e7b4a2e9207 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 17 Oct 2023 23:20:46 -0400 Subject: [PATCH 03/16] Add twitter thread api (WIP) --- src/api/status.ts | 20 +- src/api/user.ts | 8 +- src/constants.ts | 1 + src/embed/status.ts | 12 +- src/providers/twitter/processor.ts | 211 ++++++++++++++++++++ src/providers/twitter/status.ts | 308 +++++++++++++++++++++++++++++ src/types/twitterTypes.d.ts | 69 +++++-- src/types/types.d.ts | 51 ++--- src/worker.ts | 2 + 9 files changed, 627 insertions(+), 55 deletions(-) create mode 100644 src/providers/twitter/processor.ts create mode 100644 src/providers/twitter/status.ts diff --git a/src/api/status.ts b/src/api/status.ts index 76a6e8c..ef129c7 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -56,6 +56,7 @@ const populateTweetProperties = async ( name: apiUser.name, screen_name: apiUser.screen_name, avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', + // @ts-expect-error Legacy api shit avatar_color: null, banner_url: apiUser.banner_url || '', description: apiUser.description || '', @@ -72,8 +73,11 @@ const populateTweetProperties = async ( }; apiTweet.replies = tweet.legacy.reply_count; apiTweet.retweets = tweet.legacy.retweet_count; + apiTweet.reposts = tweet.legacy.retweet_count; apiTweet.likes = tweet.legacy.favorite_count; + // @ts-expect-error Legacy api shit apiTweet.color = null; + // @ts-expect-error legacy api apiTweet.twitter_card = 'tweet'; apiTweet.created_at = tweet.legacy.created_at; apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; @@ -126,11 +130,11 @@ const populateTweetProperties = async ( apiTweet.media.all.push(mediaObject); if (mediaObject.type === 'photo') { - apiTweet.twitter_card = 'summary_large_image'; + apiTweet.embed_card = 'summary_large_image'; apiTweet.media.photos = apiTweet.media.photos || []; apiTweet.media.photos.push(mediaObject); } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { - apiTweet.twitter_card = 'player'; + apiTweet.embed_card = 'player'; apiTweet.media.videos = apiTweet.media.videos || []; apiTweet.media.videos.push(mediaObject); } else { @@ -178,9 +182,9 @@ const populateTweetProperties = async ( /* Workaround: Force player card by default for videos */ /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' The mediaList however can set it to something else. TODO: Reimplement as enums */ - // @ts-expect-error see above comment - if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') { - apiTweet.twitter_card = 'player'; + + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; } /* If a language is specified in API or by user, let's try translating it! */ @@ -291,9 +295,9 @@ export const statusAPI = async ( const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet; - /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.twitter_card === 'tweet') { - apiTweet.twitter_card = apiTweet.quote.twitter_card; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet') { + apiTweet.embed_card = apiTweet.quote.embed_card; } } diff --git a/src/api/user.ts b/src/api/user.ts index 45885eb..499900c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.following = user.legacy.friends_count; apiUser.likes = user.legacy.favourites_count; apiUser.tweets = user.legacy.statuses_count; + apiUser.posts = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description); - apiUser.location = user.legacy.location; - apiUser.banner_url = user.legacy.profile_banner_url; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` + apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null; + apiUser.location = user.legacy.location ? user.legacy.location : null; + apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/constants.ts b/src/constants.ts index 5c8ff7e..addbee0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,7 @@ export const Constants = { RELEASE_NAME: RELEASE_NAME, API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`, TWITTER_ROOT: 'https://twitter.com', + TWITTER_GLOBAL_NAME_ROOT: 'twitter.com', TWITTER_API_ROOT: 'https://api.twitter.com', BOT_UA_REGEX: /bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi, diff --git a/src/embed/status.ts b/src/embed/status.ts index 1bab6d8..7f7df01 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -120,7 +120,7 @@ export const handleStatus = async ( const headers = [ ``, ``, - ``, + ``, ``, ``, `` @@ -199,8 +199,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card === 'player') { - tweet.twitter_card = 'summary_large_image'; + if (tweet.embed_card === 'player') { + tweet.embed_card = 'summary_large_image'; } break; case 'video': @@ -216,8 +216,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card !== 'player') { - tweet.twitter_card = 'player'; + if (tweet.embed_card !== 'player') { + tweet.embed_card = 'player'; } /* This Tweet has a video to render. */ break; @@ -344,7 +344,7 @@ export const handleStatus = async ( and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */ const text = useIV ? sanitizeText(newText).replace(/\n/g, '
') : sanitizeText(newText); - const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card; + const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card; /* Push basic headers relating to author, Tweet text, and site name */ headers.push( diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts new file mode 100644 index 0000000..d28203b --- /dev/null +++ b/src/providers/twitter/processor.ts @@ -0,0 +1,211 @@ +import { renderCard } from '../../helpers/card'; +import { Constants } from '../../constants'; +import { linkFixer } from '../../helpers/linkFixer'; +import { handleMosaic } from '../../helpers/mosaic'; +// import { translateTweet } from '../../helpers/translate'; +import { unescapeText } from '../../helpers/utils'; +import { processMedia } from '../../helpers/media'; +import { convertToApiUser } from '../../api/user'; + +export const buildAPITweet = async ( + tweet: GraphQLTweet, + language: string | undefined, + threadPiece = false, + legacyAPI = false + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const apiTweet = {} as APITweet; + + /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. + It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ + + if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { + tweet = tweet.result; + } + + if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { + tweet.core = tweet.tweet.core; + } + + if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { + tweet.legacy = tweet.tweet?.legacy; + } + + if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { + tweet.views = tweet?.tweet?.views; + } + + const graphQLUser = tweet.core.user_results.result; + const apiUser = convertToApiUser(graphQLUser); + + /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ + const id = tweet.rest_id ?? tweet.legacy.id_str; + + /* Populating a lot of the basics */ + apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; + apiTweet.id = id; + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); + if (!threadPiece) { + apiTweet.author = { + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, + global_screen_name: apiUser.global_screen_name, + avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, + banner_url: apiUser.banner_url ?? null, + description: apiUser.description ?? null, + location: apiUser.location ?? null, + url: apiUser.url ?? null, + followers: apiUser.followers, + following: apiUser.following, + joined: apiUser.joined, + posts: apiUser.tweets, + likes: apiUser.likes, + protected: apiUser.protected, + birthday: apiUser.birthday, + website: apiUser.website + }; + } + apiTweet.replies = tweet.legacy.reply_count; + if (legacyAPI) { + apiTweet.retweets = tweet.legacy.retweet_count; + } else { + apiTweet.reposts = tweet.legacy.retweet_count; + } + apiTweet.likes = tweet.legacy.favorite_count; + apiTweet.embed_card = 'tweet'; + apiTweet.created_at = tweet.legacy.created_at; + apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; + + apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; + + if (tweet.views.state === 'EnabledWithCount') { + apiTweet.views = parseInt(tweet.views.count || '0') ?? null; + } else { + apiTweet.views = null; + } + console.log('note_tweet', JSON.stringify(tweet.note_tweet)); + const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; + + if (noteTweetText) { + tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; + tweet.legacy.entities.hashtags = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; + tweet.legacy.entities.symbols = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; + + console.log('We meet the conditions to use new note tweets'); + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); + apiTweet.is_note_tweet = true; + } else { + apiTweet.is_note_tweet = false; + } + + if (tweet.legacy.lang !== 'unk') { + apiTweet.lang = tweet.legacy.lang; + } else { + apiTweet.lang = null; + } + + if (legacyAPI) { + apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; + } else if (tweet.legacy.in_reply_to_screen_name) { + apiTweet.reply_of = { + screen_name: tweet.legacy.in_reply_to_screen_name || null, + post: tweet.legacy.in_reply_to_status_id_str || null + }; + } else { + apiTweet.reply_of = null; + } + + apiTweet.media = { + all: [], + photos: [], + videos: [], + }; + + const mediaList = Array.from( + tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] + ); + + // console.log('tweet', JSON.stringify(tweet)); + + /* Populate this Tweet's media */ + mediaList.forEach(media => { + const mediaObject = processMedia(media); + if (mediaObject) { + apiTweet.media?.all?.push(mediaObject); + if (mediaObject.type === 'photo') { + apiTweet.embed_card = 'summary_large_image'; + apiTweet.media?.photos?.push(mediaObject); + } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { + apiTweet.embed_card = 'player'; + apiTweet.media?.videos?.push(mediaObject); + } else { + console.log('Unknown media type', mediaObject.type); + } + } + }); + + /* Grab color palette data */ + /* + if (mediaList[0]?.ext_media_color?.palette) { + apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); + } + */ + + /* Handle photos and mosaic if available */ + if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) { + const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); + if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { + apiTweet.media.mosaic = mosaic; + } + } + + // Add Tweet source but remove the link HTML tag + if (tweet.source) { + apiTweet.source = (tweet.source || '').replace( + /(.+?)<\/a>/, + '$2' + ); + } + + /* Populate a Twitter card */ + + if (tweet.card) { + const card = renderCard(tweet.card); + if (card.external_media) { + apiTweet.media = apiTweet.media ?? {}; + apiTweet.media.external = card.external_media; + } + if (card.poll) { + apiTweet.poll = card.poll; + } + } + + /* Workaround: Force player card by default for videos */ + /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' + The mediaList however can set it to something else. TODO: Reimplement as enums */ + // @ts-expect-error see above comment + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; + } + + /* If a language is specified in API or by user, let's try translating it! */ + if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { + /* TODO: Reimplement */ + // console.log(`Attempting to translate Tweet to ${language}...`); + // const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); + // if (translateAPI !== null && translateAPI?.translation) { + // apiTweet.translation = { + // text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + // source_lang: translateAPI?.sourceLanguage || '', + // target_lang: translateAPI?.destinationLanguage || '', + // source_lang_en: translateAPI?.localizedSourceLanguage || '' + // }; + // } + } + + return apiTweet; +}; \ No newline at end of file diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/status.ts new file mode 100644 index 0000000..f43c141 --- /dev/null +++ b/src/providers/twitter/status.ts @@ -0,0 +1,308 @@ +import { IRequest } from "itty-router"; +import { Constants } from "../../constants"; +import { twitterFetch } from "../../fetch"; +import { buildAPITweet } from "./processor"; + +type GraphQLProcessBucket = { + tweets: GraphQLTweet[]; + cursors: GraphQLTimelineCursor[]; +} + +type SocialThread = { + post: APIPost | APITweet | null; + thread: (APIPost | APITweet)[] | null; + author: APIUser | null; +} + +export const fetchTwitterThread = async ( + status: string, + event: FetchEvent, + useElongator = typeof TwitterProxy !== 'undefined', + cursor: string | null = null +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( + JSON.stringify({ + focalTweetId: status, + referrer: "home", + with_rux_injections: false, + includePromotedContent: false, + withCommunity: true, + withBirdwatchNotes: true, + withQuickPromoteEligibilityTweetFields: false, + withVoice: false, + withV2Timeline: true, + cursor: cursor + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_home_pinned_timelines_enabled: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_media_download_video_enabled: true, + responsive_web_enhance_cards_enabled: true + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + () => { + return true; + } + )) as GraphQLTweetFoundResponse; +}; + +const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => { + const bucket: GraphQLProcessBucket = { + tweets: [], + cursors: [] + }; + instructions.forEach?.(instruction => { + if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { + // @ts-expect-error Use entries or moduleItems depending on the type + (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => { + const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry + const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + if (content.__typename === 'TimelineTimelineItem') { + const itemContentType = content.itemContent?.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = content.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); + } + } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') { + content.items.forEach((item) => { + const itemContentType = item.item.itemContent.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = item.item.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); + } + }); + } + }); + } + }) + + return bucket; +} + +const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { + return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; +} + +const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { + return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); +} + +const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { + const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); + if (!tweet) { + console.log('uhhh, we could not even find that tweet, dunno how that happened'); + return -1; + } + const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); + if (index === -1) { + console.log('could not find shit for', id) + console.log(bucket.cursors) + } + return index; +} + +const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { + /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ + return oldCursors.map(cursor => { + const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); + if (newCursor) { + return newCursor; + } + return cursor; + }); +} + +const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { + return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) +} + +export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise => { + const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse; + + if (!response.data) { + return { post: null, thread: null, author: null }; + } + + const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); + const originalTweet = findTweetInBucket(id, bucket); + + /* Don't bother processing thread on a null tweet */ + if (originalTweet === null) { + return { post: null, thread: null, author: null }; + } + + const post = await buildAPITweet(originalTweet, undefined, false, false); + const author = post.author; + /* remove post.author */ + // @ts-expect-error lmao + delete post.author; + + /* If we're not processing threads, let's be done here */ + if (!processThread) { + return { post: post, thread: null, author: author }; + } + + const threadTweets = [originalTweet]; + bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet); + + let currentId = id; + + /* Process tweets that are following the current one in the thread */ + while (findNextTweet(currentId, bucket) !== -1) { + const index = findNextTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.push(tweet); + + currentId = newCurrentId; + + console.log('Current index', index, 'of', bucket.tweets.length) + + /* Reached the end of the current list of tweets in thread) */ + if (index >= (bucket.tweets.length - 1)) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination down') + break; + } + console.log('Cursor present, fetching more tweets down'); + + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet)); + /* Remove old cursor and add new bottom cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of next tweet:', findNextTweet(currentId, bucket)); + } + + currentId = id; + + while (findPreviousTweet(currentId, bucket) !== -1) { + const index = findPreviousTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.unshift(tweet); + + currentId = newCurrentId; + + if (index === 0) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination up') + break; + } + console.log('Cursor present, fetching more tweets up'); + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet)); + /* Remove old cursor and add new top cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + + // console.log('updated bucket of tweets', bucket.tweets); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket)); + } + + const socialThread: SocialThread = { + post: post, + thread: [], + author: author + } + + threadTweets.forEach(async (tweet) => { + socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false)); + }); + + return socialThread; +} + +export const threadAPIProvider = async (request: IRequest) => { + const { id } = request.params; + + const processedResponse = await processTwitterThread(id, true, request); + + return new Response(JSON.stringify(processedResponse), { + headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } + }) +} \ No newline at end of file diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 35bce9b..8757117 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -354,7 +354,7 @@ type GraphQLTweetLegacy = { type GraphQLTweet = { // Workaround result: GraphQLTweet; - __typename: 'Tweet' | 'TweetUnavailable'; + __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -444,37 +444,76 @@ type TweetTombstone = { }; }; }; + +type GraphQLTimelineTweet = { + item: 'TimelineTweet'; + __typename: 'TimelineTweet'; + tweet_results: { + result: GraphQLTweet | TweetTombstone; + }; +} + +type GraphQLTimelineCursor = { + cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; + itemType: 'TimelineTimelineCursor'; + value: string; + __typename: 'TimelineTimelineCursor'; +} + +interface GraphQLBaseTimeline { + entryType: string; + __typename: string; +} + +type GraphQLTimelineItem = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineItem'; + __typename: 'TimelineTimelineItem'; + itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; +} + +type GraphQLTimelineModule = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineModule'; + __typename: 'TimelineTimelineModule'; + items: { + entryId: `conversationthread-${number}-tweet-${number}`; + item: GraphQLTimelineItem + }[]; +} + type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ entryId: `tweet-${number}`; // "tweet-1674824189176590336" sortIndex: string; - content: { - entryType: 'TimelineTimelineItem'; - __typename: 'TimelineTimelineItem'; - itemContent: { - item: 'TimelineTweet'; - __typename: 'TimelineTweet'; - tweet_results: { - result: GraphQLTweet | TweetTombstone; - }; - }; - }; + content: GraphQLTimelineItem; }; + +type GraphQLModuleTweetEntry = { + /** The entryID contains the tweet ID */ + sortIndex: string; + item: GraphQLTimelineItem | GraphQLTimelineModule; +}; + type GraphQLConversationThread = { entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336" sortIndex: string; + content: GraphQLTimelineModule; }; type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction; +type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; -type TimeLineAddEntriesInstruction = { +type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; entries: GraphQLTimelineEntry[]; }; -type TimeLineTerminateTimelineInstruction = { +type TimelineAddModulesInstruction = { + type: 'TimelineAddToModule'; + moduleItems: GraphQLTimelineEntry[]; +}; + +type TimelineTerminateTimelineInstruction = { type: 'TimelineTerminateTimeline'; direction: 'Top'; }; diff --git a/src/types/types.d.ts b/src/types/types.d.ts index d6f9f7f..e911d81 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -81,14 +81,6 @@ interface APITranslate { target_lang: string; } -interface BaseUser { - id?: string; - name?: string; - screen_name?: string; - avatar_url?: string; - banner_url?: string; -} - interface APIExternalMedia { type: 'video'; url: string; @@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia { }; } -interface APITweet { +interface APIPost { id: string; url: string; text: string; @@ -144,15 +136,11 @@ interface APITweet { created_timestamp: number; likes: number; - retweets: number; + reposts: number; replies: number; - views?: number | null; - color: string | null; - - quote?: APITweet; + quote?: APIPost; poll?: APIPoll; - translation?: APITranslate; author: APIUser; media?: { @@ -169,24 +157,41 @@ interface APITweet { replying_to: string | null; replying_to_status: string | null; - source: string; + reply_of: { + screen_name: string | null; + post: string | null; + } | null - is_note_tweet: boolean; + source: string | null; - twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; + embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; } -interface APIUser extends BaseUser { +interface APITweet extends APIPost { + retweets: number; + views?: number | null; + translation?: APITranslate; + + is_note_tweet: boolean; +} + +interface APIUser { + id: string; + name: string; + screen_name: string; + global_screen_name: string; + avatar_url: string | null; + banner_url: string | null; // verified: 'legacy' | 'blue'| 'business' | 'government'; // verified_label: string; - description: string; - location: string; + description: string | null; + location: string | null; url: string; - avatar_color?: string | null; protected: boolean; followers: number; following: number; - tweets: number; + tweets?: number; + posts?: number; likes: number; joined: string; website: { diff --git a/src/worker.ts b/src/worker.ts index f3435f9..83f3d18 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,6 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; +import { threadAPIProvider } from './providers/twitter/status'; declare const globalThis: { fetchCompletedTime: number; @@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); +router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses) From 8ce632308e502291a560a09d7de17cb551ee6b6a Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 19 Oct 2023 01:24:55 -0400 Subject: [PATCH 04/16] upstream fixes --- src/api/status.ts | 7 ++++++- src/providers/twitter/processor.ts | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/api/status.ts b/src/api/status.ts index ef129c7..9a8d4f5 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -16,7 +16,7 @@ const populateTweetProperties = async ( conversation: TweetResultsByRestIdResult, // TimelineBlobPartial, language: string | undefined // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { const apiTweet = {} as APITweet; /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. @@ -38,6 +38,11 @@ const populateTweetProperties = async ( tweet.views = tweet?.tweet?.views; } + if (typeof tweet.core === 'undefined') { + console.log('Tweet still not valid', tweet); + return null; + } + /* With v2 conversation API we re-add the user object ot the tweet because Twitter stores it separately in the conversation API. This is to consolidate it in case a user appears multiple times in a thread. */ diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index d28203b..3b8e1b7 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -13,7 +13,7 @@ export const buildAPITweet = async ( threadPiece = false, legacyAPI = false // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { const apiTweet = {} as APITweet; /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. @@ -35,6 +35,11 @@ export const buildAPITweet = async ( tweet.views = tweet?.tweet?.views; } + if (typeof tweet.core === 'undefined') { + console.log('Tweet still not valid', tweet); + return null; + } + const graphQLUser = tweet.core.user_results.result; const apiUser = convertToApiUser(graphQLUser); @@ -124,6 +129,16 @@ export const buildAPITweet = async ( photos: [], videos: [], }; + + /* We found a quote tweet, let's process that too */ + const quoteTweet = tweet.quoted_status_result; + if (quoteTweet) { + apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet'&& apiTweet.quote !== null) { + apiTweet.embed_card = apiTweet.quote.embed_card; + } + } const mediaList = Array.from( tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] From d588946bb3779d045efe2aa27a3b5cf23fa4c2f0 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Sun, 22 Oct 2023 14:28:53 -0400 Subject: [PATCH 05/16] Include release name in x-powered-by --- src/constants.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/constants.ts b/src/constants.ts index addbee0..0b0bfa5 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -59,8 +59,8 @@ export const Constants = { RESPONSE_HEADERS: { 'allow': 'OPTIONS, GET, PURGE, HEAD', 'content-type': 'text/html;charset=UTF-8', - 'x-powered-by': '🏳️‍⚧️ Trans Rights', - 'cache-control': 'max-age=3600' // Can be overriden in some cases, like poll tweets + 'x-powered-by': `${RELEASE_NAME} (Trans Rights are Human Rights)`, + 'cache-control': 'max-age=3600' // Can be overriden in some cases, like unfinished poll tweets }, API_RESPONSE_HEADERS: { 'access-control-allow-origin': '*', @@ -69,8 +69,3 @@ export const Constants = { POLL_TWEET_CACHE: 'max-age=60', DEFAULT_COLOR: '#10A3FF' }; - -if (typeof TEST !== 'undefined') { - /* Undici gets angry about unicode headers, this is a workaround. */ - Constants.RESPONSE_HEADERS['x-powered-by'] = 'Trans Rights'; -} From 0fe63e6af51d96412dcc6f9e518c5eefa6e77078 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Sun, 22 Oct 2023 17:38:07 -0400 Subject: [PATCH 06/16] more fixes --- src/api/status.ts | 4 ++-- src/helpers/quote.ts | 2 +- src/providers/twitter/processor.ts | 27 ++++++++++++--------------- src/providers/twitter/status.ts | 5 +++++ 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/api/status.ts b/src/api/status.ts index 7913b0a..028d6df 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -301,8 +301,8 @@ export const statusAPI = async ( if (quoteTweet) { apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet; /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.twitter_card === 'tweet' && apiTweet.quote !== null) { - apiTweet.twitter_card = apiTweet.quote.twitter_card; + if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { + apiTweet.embed_card = apiTweet.quote.embed_card; } } diff --git a/src/helpers/quote.ts b/src/helpers/quote.ts index f72a009..d1b4a0a 100644 --- a/src/helpers/quote.ts +++ b/src/helpers/quote.ts @@ -1,7 +1,7 @@ import { Strings } from '../strings'; /* Helper for Quote Tweets */ -export const handleQuote = (quote: APITweet): string | null => { +export const handleQuote = (quote: APIPost): string | null => { console.log('Quoting status ', quote.id); let str = `\n`; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index 3b8e1b7..a2816af 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -6,6 +6,7 @@ import { handleMosaic } from '../../helpers/mosaic'; import { unescapeText } from '../../helpers/utils'; import { processMedia } from '../../helpers/media'; import { convertToApiUser } from '../../api/user'; +import { translateTweet } from '../../helpers/translate'; export const buildAPITweet = async ( tweet: GraphQLTweet, @@ -135,7 +136,7 @@ export const buildAPITweet = async ( if (quoteTweet) { apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet; /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.embed_card === 'tweet'&& apiTweet.quote !== null) { + if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { apiTweet.embed_card = apiTweet.quote.embed_card; } } @@ -199,10 +200,6 @@ export const buildAPITweet = async ( } } - /* Workaround: Force player card by default for videos */ - /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' - The mediaList however can set it to something else. TODO: Reimplement as enums */ - // @ts-expect-error see above comment if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { apiTweet.embed_card = 'player'; } @@ -210,16 +207,16 @@ export const buildAPITweet = async ( /* If a language is specified in API or by user, let's try translating it! */ if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { /* TODO: Reimplement */ - // console.log(`Attempting to translate Tweet to ${language}...`); - // const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); - // if (translateAPI !== null && translateAPI?.translation) { - // apiTweet.translation = { - // text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), - // source_lang: translateAPI?.sourceLanguage || '', - // target_lang: translateAPI?.destinationLanguage || '', - // source_lang_en: translateAPI?.localizedSourceLanguage || '' - // }; - // } + console.log(`Attempting to translate Tweet to ${language}...`); + const translateAPI = await translateTweet(tweet, '', language); + if (translateAPI !== null && translateAPI?.translation) { + apiTweet.translation = { + text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + source_lang: translateAPI?.sourceLanguage || '', + target_lang: translateAPI?.destinationLanguage || '', + source_lang_en: translateAPI?.localizedSourceLanguage || '' + }; + } } return apiTweet; diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/status.ts index f43c141..40375f2 100644 --- a/src/providers/twitter/status.ts +++ b/src/providers/twitter/status.ts @@ -172,6 +172,11 @@ export const processTwitterThread = async (id: string, processThread = false, re } const post = await buildAPITweet(originalTweet, undefined, false, false); + + if (post === null) { + return { post: null, thread: null, author: null }; + } + const author = post.author; /* remove post.author */ // @ts-expect-error lmao From 40aa057909cf96985881074e0a7a66cbf0f92eab Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Mon, 30 Oct 2023 18:09:34 -0400 Subject: [PATCH 07/16] Update wrangler.example.toml --- wrangler.example.toml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/wrangler.example.toml b/wrangler.example.toml index 3df432e..6def706 100644 --- a/wrangler.example.toml +++ b/wrangler.example.toml @@ -3,15 +3,16 @@ account_id = "[CLOUDFLARE_ACCOUNT_ID]" main = "./dist/worker.js" compatibility_date = "2023-08-15" send_metrics = false -services = [ - { binding = "TwitterProxy", service = "elongator" } -] + +# Remove this if not using Cloudflare Analytics Engine analytics_engine_datasets = [ { binding = "AnalyticsEngine" } ] -[build] -command = "npm run build" +# Remove this if not using elongator account proxying +services = [ + { binding = "TwitterProxy", service = "elongator" } +] -[miniflare.globals] -TEST = "true" # Will have unicode character errors in headers if not set to true and running unit tests \ No newline at end of file +[build] +command = "npm run build" \ No newline at end of file From ae7c432481b489590a8fab9081ed4fec726183a3 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 31 Oct 2023 18:38:08 -0400 Subject: [PATCH 08/16] WIP embed rework --- src/api/status.ts | 2 +- src/embed/status.ts | 41 +++- src/experiments.ts | 10 +- src/helpers/author.ts | 12 +- .../twitter/{status.ts => conversation.ts} | 175 ++++++++++++++---- src/providers/twitter/processor.ts | 2 +- src/render/instantview.ts | 4 +- src/render/photo.ts | 2 + src/types/twitterTypes.d.ts | 23 ++- src/types/types.d.ts | 38 ++-- src/worker.ts | 2 +- 11 files changed, 230 insertions(+), 81 deletions(-) rename src/providers/twitter/{status.ts => conversation.ts} (65%) diff --git a/src/api/status.ts b/src/api/status.ts index 028d6df..1f5bd49 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -77,7 +77,7 @@ const populateTweetProperties = async ( website: apiUser.website }; apiTweet.replies = tweet.legacy.reply_count; - apiTweet.retweets = tweet.legacy.retweet_count; + apiTweet.reposts = tweet.legacy.retweet_count; apiTweet.reposts = tweet.legacy.retweet_count; apiTweet.likes = tweet.legacy.favorite_count; // @ts-expect-error Legacy api shit diff --git a/src/embed/status.ts b/src/embed/status.ts index 3acdb42..ef644fc 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -3,10 +3,11 @@ import { handleQuote } from '../helpers/quote'; import { formatNumber, sanitizeText, truncateWithEllipsis } from '../helpers/utils'; import { Strings } from '../strings'; import { getAuthorText } from '../helpers/author'; -import { statusAPI } from '../api/status'; import { renderPhoto } from '../render/photo'; import { renderVideo } from '../render/video'; import { renderInstantView } from '../render/instantview'; +import { constructTwitterThread } from '../providers/twitter/conversation'; +import { IRequest } from 'itty-router'; export const returnError = (error: string): StatusResponse => { return { @@ -33,8 +34,38 @@ export const handleStatus = async ( ): Promise => { console.log('Direct?', flags?.direct); - const api = await statusAPI(status, language, event as FetchEvent, flags); - const tweet = api?.tweet as APITweet; + const request = (event as FetchEvent).request as IRequest; + let fetchWithThreads = false; + + if (request.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) { + fetchWithThreads = true; + } + + const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined); + + const tweet = thread?.post as APITweet; + + const api = { + code: thread.code, + message: '', + tweet: tweet + }; + + switch(api.code) { + case 200: + api.message = "OK"; + break; + case 401: + api.message = "PRIVATE_TWEET"; + break; + case 404: + api.message = "NOT_FOUND"; + break; + case 500: + console.log(api); + api.message = "API_FAIL"; + break; + } /* Catch this request if it's an API response */ if (flags?.api) { @@ -46,6 +77,10 @@ export const handleStatus = async ( }; } + if (tweet === null) { + return returnError(Strings.ERROR_TWEET_NOT_FOUND); + } + /* If there was any errors fetching the Tweet, we'll return it */ switch (api.code) { case 401: diff --git a/src/experiments.ts b/src/experiments.ts index b5c084c..3c437fc 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -1,6 +1,7 @@ export enum Experiment { ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT', - ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API' + ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API', + TWEET_DETAIL_API = 'TWEET_DETAIL_API', } type ExperimentConfig = { @@ -19,7 +20,12 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = { name: 'Elongator profile API', description: 'Use Elongator to load profiles', percentage: 0 - } + }, + [Experiment.TWEET_DETAIL_API]: { + name: 'Tweet detail API', + description: 'Use Tweet Detail API (where available with elongator)', + percentage: 0.75 + }, }; export const experimentCheck = (experiment: Experiment, condition = true) => { diff --git a/src/helpers/author.ts b/src/helpers/author.ts index bf0a4a8..dde6c44 100644 --- a/src/helpers/author.ts +++ b/src/helpers/author.ts @@ -3,13 +3,13 @@ import { formatNumber } from './utils'; /* The embed "author" text we populate with replies, retweets, and likes unless it's a video */ export const getAuthorText = (tweet: APITweet): string | null => { /* Build out reply, retweet, like counts */ - if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) { + if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) { let authorText = ''; if (tweet.replies > 0) { authorText += `${formatNumber(tweet.replies)} 💬 `; } - if (tweet.retweets > 0) { - authorText += `${formatNumber(tweet.retweets)} 🔁 `; + if (tweet.reposts > 0) { + authorText += `${formatNumber(tweet.reposts)} 🔁 `; } if (tweet.likes > 0) { authorText += `${formatNumber(tweet.likes)} ❤️ `; @@ -28,13 +28,13 @@ export const getAuthorText = (tweet: APITweet): string | null => { /* The embed "author" text we populate with replies, retweets, and likes unless it's a video */ export const getSocialTextIV = (tweet: APITweet): string | null => { /* Build out reply, retweet, like counts */ - if (tweet.likes > 0 || tweet.retweets > 0 || tweet.replies > 0) { + if (tweet.likes > 0 || tweet.reposts > 0 || tweet.replies > 0) { let authorText = ''; if (tweet.replies > 0) { authorText += `💬 ${formatNumber(tweet.replies)} `; } - if (tweet.retweets > 0) { - authorText += `🔁 ${formatNumber(tweet.retweets)} `; + if (tweet.reposts > 0) { + authorText += `🔁 ${formatNumber(tweet.reposts)} `; } if (tweet.likes > 0) { authorText += `❤️ ${formatNumber(tweet.likes)} `; diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/conversation.ts similarity index 65% rename from src/providers/twitter/status.ts rename to src/providers/twitter/conversation.ts index 40375f2..dd7ce3e 100644 --- a/src/providers/twitter/status.ts +++ b/src/providers/twitter/conversation.ts @@ -2,19 +2,10 @@ import { IRequest } from "itty-router"; import { Constants } from "../../constants"; import { twitterFetch } from "../../fetch"; import { buildAPITweet } from "./processor"; +import { Experiment, experimentCheck } from "../../experiments"; +import { isGraphQLTweet } from "../../helpers/graphql"; -type GraphQLProcessBucket = { - tweets: GraphQLTweet[]; - cursors: GraphQLTimelineCursor[]; -} - -type SocialThread = { - post: APIPost | APITweet | null; - thread: (APIPost | APITweet)[] | null; - author: APIUser | null; -} - -export const fetchTwitterThread = async ( +export const fetchTweetDetail = async ( status: string, event: FetchEvent, useElongator = typeof TwitterProxy !== 'undefined', @@ -66,13 +57,102 @@ export const fetchTwitterThread = async ( )}`, event, useElongator, - () => { + (_conversation: unknown) => { + const conversation = _conversation as GraphQLTweetFoundResponse; + const tweet = findTweetInBucket(status, processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions)); + if (tweet && isGraphQLTweet(tweet)) { return true; + } + console.log('invalid graphql tweet'); + + return Array.isArray(conversation?.errors); } )) as GraphQLTweetFoundResponse; }; -const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => { +export const fetchByRestId = async ( + status: string, + event: FetchEvent, + useElongator = experimentCheck( + Experiment.ELONGATOR_BY_DEFAULT, + typeof TwitterProxy !== 'undefined' + ) +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( + JSON.stringify({ + tweetId: status, + withCommunity: false, + includePromotedContent: false, + withVoice: false + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + creator_subscriptions_tweet_preview_api_enabled: true, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_media_download_video_enabled: false, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_enhance_cards_enabled: false + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + (_conversation: unknown) => { + const conversation = _conversation as TweetResultsByRestIdResult; + // If we get a not found error it's still a valid response + const tweet = conversation.data?.tweetResult?.result; + if (isGraphQLTweet(tweet)) { + return true; + } + console.log('invalid graphql tweet'); + if ( + !tweet && + typeof conversation.data?.tweetResult === 'object' && + Object.keys(conversation.data?.tweetResult || {}).length === 0 + ) { + console.log('tweet was not found'); + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { + console.log('tweet is nsfw'); + return true; + } + if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + console.log('tweet is protected'); + return true; + } + if (tweet?.__typename === 'TweetUnavailable') { + console.log('generic tweet unavailable error'); + return true; + } + // Final clause for checking if it's valid is if there's errors + return Array.isArray(conversation.errors); + } + )) as TweetResultsByRestIdResult; +}; + + +const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => { const bucket: GraphQLProcessBucket = { tweets: [], cursors: [] @@ -133,12 +213,7 @@ const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => console.log('uhhh, we could not even find that tweet, dunno how that happened'); return -1; } - const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); - if (index === -1) { - console.log('could not find shit for', id) - console.log(bucket.cursors) - } - return index; + return bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); } const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { @@ -156,11 +231,39 @@ const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) } -export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise => { - const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse; +export const constructTwitterThread = async (id: string, + processThread = false, + request: IRequest, + language: string | undefined, + legacyAPI = false): Promise => { - if (!response.data) { - return { post: null, thread: null, author: null }; + let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult; + let post: APITweet; + + if (typeof TwitterProxy === "undefined" || !experimentCheck(Experiment.TWEET_DETAIL_API)) { + console.log('Using TweetResultsByRestId for request...'); + response = await fetchByRestId(id, request.event) as TweetResultsByRestIdResult; + + const result = response?.data?.tweetResult?.result as GraphQLTweet; + + if (typeof result?.tweet === "undefined") { + return { post: null, thread: null, author: null, code: 404 }; + } + + post = await buildAPITweet(result, language, false, legacyAPI) as APITweet; + + if (post === null) { + return { post: null, thread: null, author: null, code: 404 }; + } + + return { post: post, thread: null, author: post.author, code: 200 }; + } else { + console.log('Using TweetDetail for request...'); + response = await fetchTweetDetail(id, request.event) as GraphQLTweetFoundResponse; + + if (!response.data) { + return { post: null, thread: null, author: null, code: 404 }; + } } const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); @@ -168,23 +271,20 @@ export const processTwitterThread = async (id: string, processThread = false, re /* Don't bother processing thread on a null tweet */ if (originalTweet === null) { - return { post: null, thread: null, author: null }; + return { post: null, thread: null, author: null, code: 404 }; } - - const post = await buildAPITweet(originalTweet, undefined, false, false); + + post = await buildAPITweet(originalTweet, undefined, false, legacyAPI) as APITweet; if (post === null) { - return { post: null, thread: null, author: null }; + return { post: null, thread: null, author: null, code: 404 }; } const author = post.author; - /* remove post.author */ - // @ts-expect-error lmao - delete post.author; /* If we're not processing threads, let's be done here */ if (!processThread) { - return { post: post, thread: null, author: author }; + return { post: post, thread: null, author: author, code: 200 }; } const threadTweets = [originalTweet]; @@ -221,7 +321,7 @@ export const processTwitterThread = async (id: string, processThread = false, re let loadCursor: GraphQLTweetFoundResponse; try { - loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value) if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { console.log('Unknown data while fetching cursor', loadCursor); @@ -267,7 +367,7 @@ export const processTwitterThread = async (id: string, processThread = false, re let loadCursor: GraphQLTweetFoundResponse; try { - loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value) if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { console.log('Unknown data while fetching cursor', loadCursor); @@ -292,11 +392,12 @@ export const processTwitterThread = async (id: string, processThread = false, re const socialThread: SocialThread = { post: post, thread: [], - author: author + author: author, + code: 200 } threadTweets.forEach(async (tweet) => { - socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false)); + socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false) as APITweet); }); return socialThread; @@ -305,7 +406,7 @@ export const processTwitterThread = async (id: string, processThread = false, re export const threadAPIProvider = async (request: IRequest) => { const { id } = request.params; - const processedResponse = await processTwitterThread(id, true, request); + const processedResponse = await constructTwitterThread(id, true, request, undefined); return new Response(JSON.stringify(processedResponse), { headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index a2816af..cf61deb 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -74,6 +74,7 @@ export const buildAPITweet = async ( } apiTweet.replies = tweet.legacy.reply_count; if (legacyAPI) { + // @ts-expect-error Use retweets for legacy API apiTweet.retweets = tweet.legacy.retweet_count; } else { apiTweet.reposts = tweet.legacy.retweet_count; @@ -206,7 +207,6 @@ export const buildAPITweet = async ( /* If a language is specified in API or by user, let's try translating it! */ if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { - /* TODO: Reimplement */ console.log(`Attempting to translate Tweet to ${language}...`); const translateAPI = await translateTweet(tweet, '', language); if (translateAPI !== null && translateAPI?.translation) { diff --git a/src/render/instantview.ts b/src/render/instantview.ts index 2ffcbec..ae64b7c 100644 --- a/src/render/instantview.ts +++ b/src/render/instantview.ts @@ -4,7 +4,7 @@ import { getSocialTextIV } from '../helpers/author'; import { sanitizeText } from '../helpers/utils'; import { Strings } from '../strings'; -const populateUserLinks = (tweet: APITweet, text: string): string => { +const populateUserLinks = (tweet: APIPost, text: string): string => { /* TODO: Maybe we can add username splices to our API so only genuinely valid users are linked? */ text.match(/@(\w{1,15})/g)?.forEach(match => { const username = match.replace('@', ''); @@ -117,7 +117,7 @@ const truncateSocialCount = (count: number): string => { } }; -const generateTweetFooter = (tweet: APITweet, isQuote = false): string => { +const generateTweetFooter = (tweet: APIPost, isQuote = false): string => { const { author } = tweet; let description = author.description; diff --git a/src/render/photo.ts b/src/render/photo.ts index 8813a09..4c78824 100644 --- a/src/render/photo.ts +++ b/src/render/photo.ts @@ -35,6 +35,8 @@ export const renderPhoto = ( } } + console.log('photo!', photo); + if (photo.type === 'mosaic_photo' && !isOverrideMedia) { instructions.addHeaders = [ ``, diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 8757117..5df0992 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -501,7 +501,7 @@ type GraphQLConversationThread = { type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; +type ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; @@ -544,9 +544,10 @@ type GraphQLTweetNotFoundResponse = { data: Record; }; type GraphQLTweetFoundResponse = { + errors?: unknown[]; data: { threaded_conversation_with_injections_v2: { - instructions: V2ThreadInstruction[]; + instructions: ThreadInstruction[]; }; }; }; @@ -555,12 +556,18 @@ type TweetResultsByRestIdResult = { errors?: unknown[]; data?: { tweetResult?: { - result?: - | { - __typename: 'TweetUnavailable'; - reason: 'NsfwLoggedOut' | 'Protected'; - } - | GraphQLTweet; + result?: TweetStub | GraphQLTweet; }; }; }; + +type TweetStub = { + __typename: 'TweetUnavailable'; + reason: 'NsfwLoggedOut' | 'Protected'; +} + + +interface GraphQLProcessBucket { + tweets: GraphQLTweet[]; + cursors: GraphQLTimelineCursor[]; +} diff --git a/src/types/types.d.ts b/src/types/types.d.ts index e911d81..6beb3a1 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -43,31 +43,18 @@ interface Request { }; } -interface Size { - width: number; - height: number; -} - -interface HorizontalSize { - width: number; - height: number; - firstWidth: number; - secondWidth: number; -} - -interface VerticalSize { - width: number; - height: number; - firstHeight: number; - secondHeight: number; -} - interface TweetAPIResponse { code: number; message: string; tweet?: APITweet; } +interface SocialPostAPIResponse { + code: number; + message: string; + post?: APITweet; +} + interface UserAPIResponse { code: number; message: string; @@ -168,7 +155,6 @@ interface APIPost { } interface APITweet extends APIPost { - retweets: number; views?: number | null; translation?: APITranslate; @@ -204,3 +190,15 @@ interface APIUser { year?: number; }; } + +interface SocialPost { + post: APIPost | APITweet | null; + author: APIUser | null; +} + +interface SocialThread { + post: APIPost | APITweet | null; + thread: (APIPost | APITweet)[] | null; + author: APIUser | null; + code: number; +} \ No newline at end of file diff --git a/src/worker.ts b/src/worker.ts index 0565029..2a6e227 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,7 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; -import { threadAPIProvider } from './providers/twitter/status'; +import { threadAPIProvider } from './providers/twitter/conversation'; declare const globalThis: { fetchCompletedTime: number; From eb90f48226b899d499da1e2ed0866b9c580c36ce Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 03:26:19 -0400 Subject: [PATCH 09/16] Use proto-v2 thread API internally --- src/api/status.ts | 315 -------------------------- src/api/user.ts | 8 +- src/embed/status.ts | 4 +- src/experiments.ts | 2 +- src/providers/twitter/conversation.ts | 4 +- src/providers/twitter/processor.ts | 63 ++++-- src/render/instantview.ts | 8 +- src/types/types.d.ts | 20 +- test/index.test.ts | 2 + 9 files changed, 67 insertions(+), 359 deletions(-) delete mode 100644 src/api/status.ts diff --git a/src/api/status.ts b/src/api/status.ts deleted file mode 100644 index 1f5bd49..0000000 --- a/src/api/status.ts +++ /dev/null @@ -1,315 +0,0 @@ -import { renderCard } from '../helpers/card'; -import { Constants } from '../constants'; -import { fetchConversation } from '../fetch'; -import { linkFixer } from '../helpers/linkFixer'; -import { handleMosaic } from '../helpers/mosaic'; -import { translateTweet } from '../helpers/translate'; -import { unescapeText } from '../helpers/utils'; -import { processMedia } from '../helpers/media'; -import { convertToApiUser } from './user'; -import { isGraphQLTweet } from '../helpers/graphql'; - -/* This function does the heavy lifting of processing data from Twitter API - and using it to create FixTweet's streamlined API responses */ -const populateTweetProperties = async ( - tweet: GraphQLTweet, - conversation: TweetResultsByRestIdResult, // TimelineBlobPartial, - language: string | undefined - // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { - const apiTweet = {} as APITweet; - - /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. - It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ - - if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { - tweet = tweet.result; - } - - if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { - tweet.core = tweet.tweet.core; - } - - if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { - tweet.legacy = tweet.tweet?.legacy; - } - - if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { - tweet.views = tweet?.tweet?.views; - } - - if (typeof tweet.core === 'undefined') { - console.log('Tweet still not valid', tweet); - return null; - } - - /* With v2 conversation API we re-add the user object ot the tweet because - Twitter stores it separately in the conversation API. This is to consolidate - it in case a user appears multiple times in a thread. */ - const graphQLUser = tweet.core.user_results.result; - const apiUser = convertToApiUser(graphQLUser); - - /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ - const id = tweet.rest_id ?? tweet.legacy.id_str; - - /* Populating a lot of the basics */ - apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; - apiTweet.id = id; - apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); - apiTweet.author = { - id: apiUser.id, - name: apiUser.name, - screen_name: apiUser.screen_name, - avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', - // @ts-expect-error Legacy api shit - avatar_color: null, - banner_url: apiUser.banner_url || '', - description: apiUser.description || '', - location: apiUser.location || '', - url: apiUser.url || '', - followers: apiUser.followers, - following: apiUser.following, - joined: apiUser.joined, - tweets: apiUser.tweets, - likes: apiUser.likes, - protected: apiUser.protected, - birthday: apiUser.birthday, - website: apiUser.website - }; - apiTweet.replies = tweet.legacy.reply_count; - apiTweet.reposts = tweet.legacy.retweet_count; - apiTweet.reposts = tweet.legacy.retweet_count; - apiTweet.likes = tweet.legacy.favorite_count; - // @ts-expect-error Legacy api shit - apiTweet.color = null; - // @ts-expect-error legacy api - apiTweet.twitter_card = 'tweet'; - apiTweet.created_at = tweet.legacy.created_at; - apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; - - apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; - - if (tweet.views.state === 'EnabledWithCount') { - apiTweet.views = parseInt(tweet.views.count || '0') ?? null; - } else { - apiTweet.views = null; - } - console.log('note_tweet', JSON.stringify(tweet.note_tweet)); - const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; - - if (noteTweetText) { - tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; - tweet.legacy.entities.hashtags = - tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; - tweet.legacy.entities.symbols = - tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; - - console.log('We meet the conditions to use new note tweets'); - apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); - apiTweet.is_note_tweet = true; - } else { - apiTweet.is_note_tweet = false; - } - - if (tweet.legacy.lang !== 'unk') { - apiTweet.lang = tweet.legacy.lang; - } else { - apiTweet.lang = null; - } - - apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; - apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; - - const mediaList = Array.from( - tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] - ); - - // console.log('tweet', JSON.stringify(tweet)); - - /* Populate this Tweet's media */ - mediaList.forEach(media => { - const mediaObject = processMedia(media); - if (mediaObject) { - apiTweet.media = apiTweet.media || {}; - apiTweet.media.all = apiTweet.media?.all || []; - apiTweet.media.all.push(mediaObject); - - if (mediaObject.type === 'photo') { - apiTweet.embed_card = 'summary_large_image'; - apiTweet.media.photos = apiTweet.media.photos || []; - apiTweet.media.photos.push(mediaObject); - } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { - apiTweet.embed_card = 'player'; - apiTweet.media.videos = apiTweet.media.videos || []; - apiTweet.media.videos.push(mediaObject); - } else { - console.log('Unknown media type', mediaObject.type); - } - } - }); - - /* Grab color palette data */ - /* - if (mediaList[0]?.ext_media_color?.palette) { - apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); - } - */ - - /* Handle photos and mosaic if available */ - if ((apiTweet?.media?.photos?.length || 0) > 1) { - const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); - if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { - apiTweet.media.mosaic = mosaic; - } - } - - // Add Tweet source but remove the link HTML tag - if (tweet.source) { - apiTweet.source = (tweet.source || '').replace( - /(.+?)<\/a>/, - '$2' - ); - } - - /* Populate a Twitter card */ - - if (tweet.card) { - const card = renderCard(tweet.card); - if (card.external_media) { - apiTweet.media = apiTweet.media || {}; - apiTweet.media.external = card.external_media; - } - if (card.poll) { - apiTweet.poll = card.poll; - } - } - - /* Workaround: Force player card by default for videos */ - /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' - The mediaList however can set it to something else. TODO: Reimplement as enums */ - - if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { - apiTweet.embed_card = 'player'; - } - - /* If a language is specified in API or by user, let's try translating it! */ - if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { - console.log(`Attempting to translate Tweet to ${language}...`); - const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); - if (translateAPI !== null && translateAPI?.translation) { - apiTweet.translation = { - text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), - source_lang: translateAPI?.sourceLanguage || '', - target_lang: translateAPI?.destinationLanguage || '', - source_lang_en: translateAPI?.localizedSourceLanguage || '' - }; - } - } - - return apiTweet; -}; - -const writeDataPoint = ( - event: FetchEvent, - language: string | undefined, - nsfw: boolean, - returnCode: string, - flags?: InputFlags -) => { - console.log('Writing data point...'); - if (typeof AnalyticsEngine !== 'undefined') { - const flagString = - Object.keys(flags || {}) - // @ts-expect-error - TypeScript doesn't like iterating over the keys, but that's OK - .filter(flag => flags?.[flag])[0] || 'standard'; - - AnalyticsEngine.writeDataPoint({ - blobs: [ - event.request.cf?.colo as string /* Datacenter location */, - event.request.cf?.country as string /* Country code */, - event.request.headers.get('user-agent') ?? - '' /* User agent (for aggregating bots calling) */, - returnCode /* Return code */, - flagString /* Type of request */, - language ?? '' /* For translate feature */ - ], - doubles: [nsfw ? 1 : 0 /* NSFW media = 1, No NSFW Media = 0 */], - indexes: [event.request.headers.get('cf-ray') ?? '' /* CF Ray */] - }); - } -}; - -/* API for Twitter statuses (Tweets) - Used internally by FixTweet's embed service, or - available for free using api.fxtwitter.com. */ -export const statusAPI = async ( - status: string, - language: string | undefined, - event: FetchEvent, - flags?: InputFlags -): Promise => { - const res = await fetchConversation(status, event); - const tweet = res.data?.tweetResult?.result; - if (!tweet) { - return { code: 404, message: 'NOT_FOUND' }; - } - /* We're handling this in the actual fetch code now */ - - // if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { - // wasMediaBlockedNSFW = true; - // res = await fetchConversation(status, event, true); - // } - - // console.log(JSON.stringify(tweet)) - - if (tweet.__typename === 'TweetUnavailable') { - if ((tweet as { reason: string })?.reason === 'Protected') { - writeDataPoint(event, language, false, 'PRIVATE_TWEET', flags); - return { code: 401, message: 'PRIVATE_TWEET' }; - // } else if (tweet.reason === 'NsfwLoggedOut') { - // // API failure as elongator should have handled this - // writeDataPoint(event, language, wasMediaBlockedNSFW, 'API_FAIL', flags); - // return { code: 500, message: 'API_FAIL' }; - } else { - // Api failure at parsing status - writeDataPoint(event, language, false, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; - } - } - // If the tweet is not a graphQL tweet something went wrong - if (!isGraphQLTweet(tweet)) { - console.log('Tweet was not a valid tweet', tweet); - writeDataPoint(event, language, false, 'API_FAIL', flags); - return { code: 500, message: 'API_FAIL' }; - } - - /* - if (tweet.retweeted_status_id_str) { - tweet = conversation?.globalObjects?.tweets?.[tweet.retweeted_status_id_str] || {}; - } - */ - - if (!tweet) { - return { code: 404, message: 'NOT_FOUND' }; - } - /* Creating the response objects */ - const response: TweetAPIResponse = { code: 200, message: 'OK' } as TweetAPIResponse; - const apiTweet: APITweet = (await populateTweetProperties(tweet, res, language)) as APITweet; - - /* We found a quote tweet, let's process that too */ - const quoteTweet = tweet.quoted_status_result; - if (quoteTweet) { - apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet; - /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { - apiTweet.embed_card = apiTweet.quote.embed_card; - } - } - - /* Finally, staple the Tweet to the response and return it */ - response.tweet = apiTweet; - - writeDataPoint(event, language, false, 'OK', flags); - - return response; -}; diff --git a/src/api/user.ts b/src/api/user.ts index 499900c..c9ba200 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -10,14 +10,14 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.followers = user.legacy.followers_count; apiUser.following = user.legacy.friends_count; apiUser.likes = user.legacy.favourites_count; + // @ts-expect-error `tweets` is only part of legacy API apiUser.tweets = user.legacy.statuses_count; - apiUser.posts = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` - apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null; - apiUser.location = user.legacy.location ? user.legacy.location : null; - apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null; + apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : ''; + apiUser.location = user.legacy.location ? user.legacy.location : ''; + apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : ''; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/embed/status.ts b/src/embed/status.ts index ef644fc..ffc930c 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -41,7 +41,7 @@ export const handleStatus = async ( fetchWithThreads = true; } - const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined); + const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined, flags?.api ?? false); const tweet = thread?.post as APITweet; @@ -395,7 +395,7 @@ export const handleStatus = async ( authorText = `↪ Replying to @${tweet.replying_to}`; /* We'll assume it's a thread if it's a reply to themselves */ } else if ( - tweet.replying_to === tweet.author.screen_name && + tweet.replying_to?.screen_name === tweet.author.screen_name && authorText === Strings.DEFAULT_AUTHOR_TEXT ) { authorText = `↪ A part of @${tweet.author.screen_name}'s thread`; diff --git a/src/experiments.ts b/src/experiments.ts index 3c437fc..f00d53a 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -14,7 +14,7 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = { [Experiment.ELONGATOR_BY_DEFAULT]: { name: 'Elongator by default', description: 'Enable Elongator by default (guest token lockout bypass)', - percentage: 0.6 + percentage: 1 }, [Experiment.ELONGATOR_PROFILE_API]: { name: 'Elongator profile API', diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index dd7ce3e..bbcbe37 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -237,6 +237,8 @@ export const constructTwitterThread = async (id: string, language: string | undefined, legacyAPI = false): Promise => { + console.log('legacyAPI', legacyAPI) + let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult; let post: APITweet; @@ -246,7 +248,7 @@ export const constructTwitterThread = async (id: string, const result = response?.data?.tweetResult?.result as GraphQLTweet; - if (typeof result?.tweet === "undefined") { + if (typeof result === "undefined") { return { post: null, thread: null, author: null, code: 404 }; } diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index cf61deb..e7b39be 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -56,16 +56,15 @@ export const buildAPITweet = async ( id: apiUser.id, name: apiUser.name, screen_name: apiUser.screen_name, - global_screen_name: apiUser.global_screen_name, avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, - banner_url: apiUser.banner_url ?? null, - description: apiUser.description ?? null, - location: apiUser.location ?? null, - url: apiUser.url ?? null, + banner_url: apiUser.banner_url, + description: apiUser.description, + location: apiUser.location, + url: apiUser.url, followers: apiUser.followers, following: apiUser.following, joined: apiUser.joined, - posts: apiUser.tweets, + posts: apiUser.posts, likes: apiUser.likes, protected: apiUser.protected, birthday: apiUser.birthday, @@ -76,8 +75,19 @@ export const buildAPITweet = async ( if (legacyAPI) { // @ts-expect-error Use retweets for legacy API apiTweet.retweets = tweet.legacy.retweet_count; + + // @ts-expect-error `tweets` is only part of legacy API + apiTweet.author.tweets = apiTweet.author.posts; + // @ts-expect-error Part of legacy API that we no longer are able to track + apiTweet.author.avatar_color = null; + // @ts-expect-error Use retweets for legacy API + delete apiTweet.reposts; + // @ts-expect-error Use tweets and not posts for legacy API + delete apiTweet.author.posts; + delete apiTweet.author.global_screen_name; } else { apiTweet.reposts = tweet.legacy.retweet_count; + apiTweet.author.global_screen_name = apiUser.global_screen_name; } apiTweet.likes = tweet.legacy.favorite_count; apiTweet.embed_card = 'tweet'; @@ -115,27 +125,25 @@ export const buildAPITweet = async ( } if (legacyAPI) { + // @ts-expect-error Use replying_to string for legacy API apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + // @ts-expect-error Use replying_to_status string for legacy API apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; } else if (tweet.legacy.in_reply_to_screen_name) { - apiTweet.reply_of = { + apiTweet.replying_to = { screen_name: tweet.legacy.in_reply_to_screen_name || null, post: tweet.legacy.in_reply_to_status_id_str || null }; } else { - apiTweet.reply_of = null; + apiTweet.replying_to = null; } - - apiTweet.media = { - all: [], - photos: [], - videos: [], - }; + + apiTweet.media = {}; /* We found a quote tweet, let's process that too */ const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { - apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet; + apiTweet.quote = (await buildAPITweet(quoteTweet, language, threadPiece, legacyAPI)) as APITweet; /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { apiTweet.embed_card = apiTweet.quote.embed_card; @@ -152,13 +160,16 @@ export const buildAPITweet = async ( mediaList.forEach(media => { const mediaObject = processMedia(media); if (mediaObject) { + apiTweet.media.all = apiTweet.media?.all ?? []; apiTweet.media?.all?.push(mediaObject); if (mediaObject.type === 'photo') { apiTweet.embed_card = 'summary_large_image'; - apiTweet.media?.photos?.push(mediaObject); + apiTweet.media.photos = apiTweet.media?.photos ?? []; + apiTweet.media.photos?.push(mediaObject); } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { apiTweet.embed_card = 'player'; - apiTweet.media?.videos?.push(mediaObject); + apiTweet.media.videos = apiTweet.media?.videos ?? []; + apiTweet.media.videos?.push(mediaObject); } else { console.log('Unknown media type', mediaObject.type); } @@ -173,7 +184,7 @@ export const buildAPITweet = async ( */ /* Handle photos and mosaic if available */ - if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) { + if ((apiTweet?.media.photos?.length || 0) > 1 && !threadPiece) { const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { apiTweet.media.mosaic = mosaic; @@ -193,7 +204,6 @@ export const buildAPITweet = async ( if (tweet.card) { const card = renderCard(tweet.card); if (card.external_media) { - apiTweet.media = apiTweet.media ?? {}; apiTweet.media.external = card.external_media; } if (card.poll) { @@ -201,7 +211,7 @@ export const buildAPITweet = async ( } } - if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + if (apiTweet.media?.videos && apiTweet.media?.videos.length > 0 && apiTweet.embed_card !== 'player') { apiTweet.embed_card = 'player'; } @@ -219,5 +229,18 @@ export const buildAPITweet = async ( } } + if (legacyAPI) { + // @ts-expect-error Use twitter_card for legacy API + apiTweet.twitter_card = apiTweet.embed_card; + // @ts-expect-error Part of legacy API that we no longer are able to track + apiTweet.color = null + // @ts-expect-error Use twitter_card for legacy API + delete apiTweet.embed_card; + if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) { + // @ts-expect-error media is not required in legacy API if empty + delete apiTweet.media; + } + } + return apiTweet; }; \ No newline at end of file diff --git a/src/render/instantview.ts b/src/render/instantview.ts index ae64b7c..3b92d3e 100644 --- a/src/render/instantview.ts +++ b/src/render/instantview.ts @@ -16,7 +16,7 @@ const populateUserLinks = (tweet: APIPost, text: string): string => { return text; }; -const generateTweetMedia = (tweet: APITweet): string => { +const generateTweetMedia = (tweet: APIPost): string => { let media = ''; if (tweet.media?.all?.length) { tweet.media.all.forEach(mediaItem => { @@ -131,7 +131,7 @@ const generateTweetFooter = (tweet: APIPost, isQuote = false): string => { {aboutSection} `.format({ - socialText: getSocialTextIV(tweet) || '', + socialText: getSocialTextIV(tweet as APITweet) || '', viewOriginal: !isQuote ? `View original post` : notApplicableComment, aboutSection: isQuote ? '' @@ -161,13 +161,13 @@ const generateTweetFooter = (tweet: APIPost, isQuote = false): string => { }); }; -const generateTweet = (tweet: APITweet, isQuote = false): string => { +const generateTweet = (tweet: APIPost, isQuote = false): string => { let text = paragraphify(sanitizeText(tweet.text), isQuote); text = htmlifyLinks(text); text = htmlifyHashtags(text); text = populateUserLinks(tweet, text); - const translatedText = getTranslatedText(tweet, isQuote); + const translatedText = getTranslatedText(tweet as APITweet, isQuote); return ` {quoteHeader} diff --git a/src/types/types.d.ts b/src/types/types.d.ts index 6beb3a1..c0fde4b 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -130,7 +130,7 @@ interface APIPost { poll?: APIPoll; author: APIUser; - media?: { + media: { external?: APIExternalMedia; photos?: APIPhoto[]; videos?: APIVideo[]; @@ -141,10 +141,7 @@ interface APIPost { lang: string | null; possibly_sensitive: boolean; - replying_to: string | null; - replying_to_status: string | null; - - reply_of: { + replying_to: { screen_name: string | null; post: string | null; } | null @@ -165,19 +162,18 @@ interface APIUser { id: string; name: string; screen_name: string; - global_screen_name: string; - avatar_url: string | null; - banner_url: string | null; + global_screen_name?: string; + avatar_url: string; + banner_url: string; // verified: 'legacy' | 'blue'| 'business' | 'government'; // verified_label: string; - description: string | null; - location: string | null; + description: string; + location: string; url: string; protected: boolean; followers: number; following: number; - tweets?: number; - posts?: number; + posts: number; likes: number; joined: string; website: { diff --git a/test/index.test.ts b/test/index.test.ts index e34f471..45185e4 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -120,8 +120,10 @@ test('API fetch basic Tweet', async () => { expect(tweet.author.avatar_url).toBeTruthy(); expect(tweet.author.banner_url).toBeTruthy(); expect(tweet.replies).toBeGreaterThan(0); + // @ts-expect-error retweets only in legacy API expect(tweet.retweets).toBeGreaterThan(0); expect(tweet.likes).toBeGreaterThan(0); + // @ts-expect-error twitter_card only in legacy API expect(tweet.twitter_card).toEqual('tweet'); expect(tweet.created_at).toEqual('Tue Mar 21 20:50:14 +0000 2006'); expect(tweet.created_timestamp).toEqual(1142974214); From fe61670e9fbc7e2ab49cbe30af8a89cab399c4aa Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 04:52:14 -0400 Subject: [PATCH 10/16] Prettier and additional fixes --- esbuild.config.mjs | 5 +- src/api/user.ts | 6 +- src/embed/status.ts | 18 +- src/experiments.ts | 6 +- src/fetch.ts | 98 ++------- src/helpers/mosaic.ts | 2 +- src/providers/twitter/conversation.ts | 276 +++++++++++++++++--------- src/providers/twitter/processor.ts | 37 +++- src/types/twitterTypes.d.ts | 21 +- src/types/types.d.ts | 8 +- src/worker.ts | 4 +- 11 files changed, 269 insertions(+), 212 deletions(-) diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 1972e50..82f22d9 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -10,7 +10,10 @@ config(); const gitCommit = execSync('git rev-parse --short HEAD').toString().trim(); const gitCommitFull = execSync('git rev-parse HEAD').toString().trim(); const gitUrl = execSync('git remote get-url origin').toString().trim(); -const gitBranch = execSync('git rev-parse --abbrev-ref HEAD').toString().trim().replace(/[\\\/]/g, '-'); +const gitBranch = execSync('git rev-parse --abbrev-ref HEAD') + .toString() + .trim() + .replace(/[\\\/]/g, '-'); let workerName = 'fixtweet'; diff --git a/src/api/user.ts b/src/api/user.ts index c9ba200..dadc83c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -14,8 +14,10 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.tweets = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` - apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : ''; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}`; + apiUser.description = user.legacy.description + ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) + : ''; apiUser.location = user.legacy.location ? user.legacy.location : ''; apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : ''; /* diff --git a/src/embed/status.ts b/src/embed/status.ts index ffc930c..8922b20 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -41,7 +41,13 @@ export const handleStatus = async ( fetchWithThreads = true; } - const thread = await constructTwitterThread(status, fetchWithThreads, request, undefined, flags?.api ?? false); + const thread = await constructTwitterThread( + status, + fetchWithThreads, + request, + undefined, + flags?.api ?? false + ); const tweet = thread?.post as APITweet; @@ -51,19 +57,19 @@ export const handleStatus = async ( tweet: tweet }; - switch(api.code) { + switch (api.code) { case 200: - api.message = "OK"; + api.message = 'OK'; break; case 401: - api.message = "PRIVATE_TWEET"; + api.message = 'PRIVATE_TWEET'; break; case 404: - api.message = "NOT_FOUND"; + api.message = 'NOT_FOUND'; break; case 500: console.log(api); - api.message = "API_FAIL"; + api.message = 'API_FAIL'; break; } diff --git a/src/experiments.ts b/src/experiments.ts index f00d53a..0021d51 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -1,7 +1,7 @@ export enum Experiment { ELONGATOR_BY_DEFAULT = 'ELONGATOR_BY_DEFAULT', ELONGATOR_PROFILE_API = 'ELONGATOR_PROFILE_API', - TWEET_DETAIL_API = 'TWEET_DETAIL_API', + TWEET_DETAIL_API = 'TWEET_DETAIL_API' } type ExperimentConfig = { @@ -24,8 +24,8 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = { [Experiment.TWEET_DETAIL_API]: { name: 'Tweet detail API', description: 'Use Tweet Detail API (where available with elongator)', - percentage: 0.75 - }, + percentage: 0.5 + } }; export const experimentCheck = (experiment: Experiment, condition = true) => { diff --git a/src/fetch.ts b/src/fetch.ts index 39c6b6a..07d19d8 100644 --- a/src/fetch.ts +++ b/src/fetch.ts @@ -1,7 +1,6 @@ import { Constants } from './constants'; import { Experiment, experimentCheck } from './experiments'; import { generateUserAgent } from './helpers/useragent'; -import { isGraphQLTweet } from './helpers/graphql'; const API_ATTEMPTS = 3; let wasElongatorDisabled = false; @@ -26,7 +25,8 @@ export const twitterFetch = async ( Experiment.ELONGATOR_BY_DEFAULT, typeof TwitterProxy !== 'undefined' ), - validateFunction: (response: unknown) => boolean + validateFunction: (response: unknown) => boolean, + elongatorRequired = false ): Promise => { let apiAttempts = 0; let newTokenGenerated = false; @@ -163,6 +163,11 @@ export const twitterFetch = async ( /* We'll usually only hit this if we get an invalid response from Twitter. It's uncommon, but it happens */ console.error('Unknown error while fetching from API', e); + /* Elongator returns strings to communicate downstream errors */ + if (String(e).indexOf('Status not found')) { + console.log('Tweet was not found'); + return {}; + } !useElongator && event && event.waitUntil(cache.delete(guestTokenRequestCacheDummy.clone(), { ignoreMethod: true })); @@ -181,7 +186,6 @@ export const twitterFetch = async ( !wasElongatorDisabled && !useElongator && typeof TwitterProxy !== 'undefined' && - // @ts-expect-error This is safe due to optional chaining (response as TweetResultsByRestIdResult)?.data?.tweetResult?.result?.reason === 'NsfwLoggedOut' ) { @@ -201,6 +205,10 @@ export const twitterFetch = async ( if (!validateFunction(response)) { console.log('Failed to fetch response, got', JSON.stringify(response)); + if (elongatorRequired) { + console.log('Elongator was required, but we failed to fetch a valid response'); + return {}; + } if (useElongator) { console.log('Elongator request failed to validate, trying again without it'); wasElongatorDisabled = true; @@ -232,87 +240,6 @@ export const twitterFetch = async ( return {}; }; -export const fetchConversation = async ( - status: string, - event: FetchEvent, - useElongator = experimentCheck( - Experiment.ELONGATOR_BY_DEFAULT, - typeof TwitterProxy !== 'undefined' - ) -): Promise => { - return (await twitterFetch( - `${ - Constants.TWITTER_ROOT - }/i/api/graphql/2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId?variables=${encodeURIComponent( - JSON.stringify({ - tweetId: status, - withCommunity: false, - includePromotedContent: false, - withVoice: false - }) - )}&features=${encodeURIComponent( - JSON.stringify({ - creator_subscriptions_tweet_preview_api_enabled: true, - tweetypie_unmention_optimization_enabled: true, - responsive_web_edit_tweet_api_enabled: true, - graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, - view_counts_everywhere_api_enabled: true, - longform_notetweets_consumption_enabled: true, - responsive_web_twitter_article_tweet_consumption_enabled: false, - tweet_awards_web_tipping_enabled: false, - freedom_of_speech_not_reach_fetch_enabled: true, - standardized_nudges_misinfo: true, - tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, - longform_notetweets_rich_text_read_enabled: true, - longform_notetweets_inline_media_enabled: true, - responsive_web_graphql_exclude_directive_enabled: true, - verified_phone_label_enabled: false, - responsive_web_media_download_video_enabled: false, - responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, - responsive_web_graphql_timeline_navigation_enabled: true, - responsive_web_enhance_cards_enabled: false - }) - )}&fieldToggles=${encodeURIComponent( - JSON.stringify({ - withArticleRichContentState: true - }) - )}`, - event, - useElongator, - (_conversation: unknown) => { - const conversation = _conversation as TweetResultsByRestIdResult; - // If we get a not found error it's still a valid response - const tweet = conversation.data?.tweetResult?.result; - if (isGraphQLTweet(tweet)) { - return true; - } - console.log('invalid graphql tweet'); - if ( - !tweet && - typeof conversation.data?.tweetResult === 'object' && - Object.keys(conversation.data?.tweetResult || {}).length === 0 - ) { - console.log('tweet was not found'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'NsfwLoggedOut') { - console.log('tweet is nsfw'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { - console.log('tweet is protected'); - return true; - } - if (tweet?.__typename === 'TweetUnavailable') { - console.log('generic tweet unavailable error'); - return true; - } - // Final clause for checking if it's valid is if there's errors - return Array.isArray(conversation.errors); - } - )) as TweetResultsByRestIdResult; -}; - export const fetchUser = async ( username: string, event: FetchEvent, @@ -359,6 +286,7 @@ export const fetchUser = async ( conversation.errors?.[0]?.code === 239) ); */ - } + }, + false )) as GraphQLUserResponse; }; diff --git a/src/helpers/mosaic.ts b/src/helpers/mosaic.ts index e686269..0e96315 100644 --- a/src/helpers/mosaic.ts +++ b/src/helpers/mosaic.ts @@ -13,7 +13,7 @@ const getDomain = (twitterId: string): string | null => { hash = (hash << 5) - hash + char; } return mosaicDomains[Math.abs(hash) % mosaicDomains.length]; -} +}; /* Handler for mosaic (multi-image combiner) */ export const handleMosaic = async ( diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index bbcbe37..3fa8956 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -1,23 +1,23 @@ -import { IRequest } from "itty-router"; -import { Constants } from "../../constants"; -import { twitterFetch } from "../../fetch"; -import { buildAPITweet } from "./processor"; -import { Experiment, experimentCheck } from "../../experiments"; -import { isGraphQLTweet } from "../../helpers/graphql"; +import { IRequest } from 'itty-router'; +import { Constants } from '../../constants'; +import { twitterFetch } from '../../fetch'; +import { buildAPITweet } from './processor'; +import { Experiment, experimentCheck } from '../../experiments'; +import { isGraphQLTweet } from '../../helpers/graphql'; export const fetchTweetDetail = async ( status: string, event: FetchEvent, useElongator = typeof TwitterProxy !== 'undefined', cursor: string | null = null -): Promise => { +): Promise => { return (await twitterFetch( `${ Constants.TWITTER_ROOT }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( JSON.stringify({ focalTweetId: status, - referrer: "home", + referrer: 'home', with_rux_injections: false, includePromotedContent: false, withCommunity: true, @@ -58,16 +58,33 @@ export const fetchTweetDetail = async ( event, useElongator, (_conversation: unknown) => { - const conversation = _conversation as GraphQLTweetFoundResponse; - const tweet = findTweetInBucket(status, processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions)); + const conversation = _conversation as TweetDetailResult; + const tweet = findTweetInBucket( + status, + processResponse(conversation.data.threaded_conversation_with_injections_v2.instructions) + ); if (tweet && isGraphQLTweet(tweet)) { return true; } - console.log('invalid graphql tweet'); + console.log('invalid graphql tweet', conversation); + const firstInstruction = ( + conversation.data?.threaded_conversation_with_injections_v2 + .instructions?.[0] as TimelineAddEntriesInstruction + )?.entries?.[0]; + if ( + ( + (firstInstruction as { content: GraphQLTimelineItem })?.content + ?.itemContent as GraphQLTimelineTweet + )?.tweet_results?.result?.__typename === 'TweetTombstone' + ) { + console.log('tweet is private'); + return true; + } return Array.isArray(conversation?.errors); - } - )) as GraphQLTweetFoundResponse; + }, + true + )) as TweetDetailResult; }; export const fetchByRestId = async ( @@ -147,11 +164,11 @@ export const fetchByRestId = async ( } // Final clause for checking if it's valid is if there's errors return Array.isArray(conversation.errors); - } + }, + false )) as TweetResultsByRestIdResult; }; - const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucket => { const bucket: GraphQLProcessBucket = { tweets: [], @@ -160,13 +177,17 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke instructions.forEach?.(instruction => { if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { // @ts-expect-error Use entries or moduleItems depending on the type - (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => { - const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry - const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + (instruction?.entries ?? instruction.moduleItems).forEach(_entry => { + const entry = _entry as + | GraphQLTimelineTweetEntry + | GraphQLConversationThread + | GraphQLModuleTweetEntry; + const content = + (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; if (content.__typename === 'TimelineTimelineItem') { const itemContentType = content.itemContent?.__typename; if (itemContentType === 'TimelineTweet') { - const entryType = content.itemContent.tweet_results.result.__typename + const entryType = content.itemContent.tweet_results.result.__typename; if (entryType === 'Tweet') { bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); } @@ -176,16 +197,20 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke } else if (itemContentType === 'TimelineTimelineCursor') { bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); } - } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') { - content.items.forEach((item) => { + } else if ( + (content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule' + ) { + content.items.forEach(item => { const itemContentType = item.item.itemContent.__typename; if (itemContentType === 'TimelineTweet') { - const entryType = item.item.itemContent.tweet_results.result.__typename + const entryType = item.item.itemContent.tweet_results.result.__typename; if (entryType === 'Tweet') { bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); } if (entryType === 'TweetWithVisibilityResults') { - bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet); + bucket.tweets.push( + item.item.itemContent.tweet_results.result.tweet as GraphQLTweet + ); } } else if (itemContentType === 'TimelineTimelineCursor') { bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); @@ -194,18 +219,18 @@ const processResponse = (instructions: ThreadInstruction[]): GraphQLProcessBucke } }); } - }) + }); return bucket; -} +}; const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; -} +}; const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); -} +}; const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); @@ -213,10 +238,15 @@ const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => console.log('uhhh, we could not even find that tweet, dunno how that happened'); return -1; } - return bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); -} + return bucket.tweets.findIndex( + _tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str + ); +}; -const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { +const consolidateCursors = ( + oldCursors: GraphQLTimelineCursor[], + newCursors: GraphQLTimelineCursor[] +): GraphQLTimelineCursor[] => { /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ return oldCursors.map(cursor => { const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); @@ -225,58 +255,87 @@ const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: Gra } return cursor; }); -} +}; const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { - return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) -} + return tweets.filter( + tweet => + tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id + ); +}; -export const constructTwitterThread = async (id: string, +/* Fetch and construct a Twitter thread */ +export const constructTwitterThread = async ( + id: string, processThread = false, request: IRequest, language: string | undefined, - legacyAPI = false): Promise => { + legacyAPI = false +): Promise => { + console.log('legacyAPI', legacyAPI); - console.log('legacyAPI', legacyAPI) - - let response: GraphQLTweetFoundResponse | TweetResultsByRestIdResult; + let response: TweetDetailResult | TweetResultsByRestIdResult | null = null; let post: APITweet; - - if (typeof TwitterProxy === "undefined" || !experimentCheck(Experiment.TWEET_DETAIL_API)) { - console.log('Using TweetResultsByRestId for request...'); - response = await fetchByRestId(id, request.event) as TweetResultsByRestIdResult; - - const result = response?.data?.tweetResult?.result as GraphQLTweet; - - if (typeof result === "undefined") { - return { post: null, thread: null, author: null, code: 404 }; - } - - post = await buildAPITweet(result, language, false, legacyAPI) as APITweet; - - if (post === null) { - return { post: null, thread: null, author: null, code: 404 }; - } - - return { post: post, thread: null, author: post.author, code: 200 }; - } else { + /* We can use TweetDetail on elongator accounts to increase per-account rate limit. + We also use TweetDetail to process threads (WIP) */ + if (typeof TwitterProxy !== 'undefined' && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { console.log('Using TweetDetail for request...'); - response = await fetchTweetDetail(id, request.event) as GraphQLTweetFoundResponse; + response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult; - if (!response.data) { + console.log(response); + + const firstInstruction = ( + response.data?.threaded_conversation_with_injections_v2 + .instructions?.[0] as TimelineAddEntriesInstruction + )?.entries?.[0]; + if ( + ( + (firstInstruction as { content: GraphQLTimelineItem })?.content + ?.itemContent as GraphQLTimelineTweet + )?.tweet_results?.result?.__typename === 'TweetTombstone' /* If a tweet is private */ + ) { + console.log('tweet is private'); + return { post: null, thread: null, author: null, code: 401 }; + } else if (!response.data) { return { post: null, thread: null, author: null, code: 404 }; } } - const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); + /* If we didn't get a response from TweetDetail we should ignore threads and try TweetResultsByRestId */ + if (!response) { + console.log('Using TweetResultsByRestId for request...'); + response = (await fetchByRestId(id, request.event)) as TweetResultsByRestIdResult; + + const result = response?.data?.tweetResult?.result as GraphQLTweet; + + if (typeof result === 'undefined') { + return { post: null, thread: null, author: null, code: 404 }; + } + + const buildPost = await buildAPITweet(result, language, false, legacyAPI); + + if ((buildPost as FetchResults).status === 401) { + return { post: null, thread: null, author: null, code: 401 }; + } else if (buildPost === null) { + return { post: null, thread: null, author: null, code: 404 }; + } + + post = buildPost as APITweet; + + return { post: post, thread: null, author: post.author, code: 200 }; + } + + const bucket = processResponse( + response.data.threaded_conversation_with_injections_v2.instructions + ); const originalTweet = findTweetInBucket(id, bucket); /* Don't bother processing thread on a null tweet */ if (originalTweet === null) { return { post: null, thread: null, author: null, code: 404 }; } - - post = await buildAPITweet(originalTweet, undefined, false, legacyAPI) as APITweet; + + post = (await buildAPITweet(originalTweet, undefined, false, legacyAPI)) as APITweet; if (post === null) { return { post: null, thread: null, author: null, code: 404 }; @@ -301,41 +360,58 @@ export const constructTwitterThread = async (id: string, const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; - console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + console.log( + 'adding next tweet to thread', + newCurrentId, + 'from', + currentId, + 'at index', + index, + 'in bucket' + ); threadTweets.push(tweet); currentId = newCurrentId; - console.log('Current index', index, 'of', bucket.tweets.length) + console.log('Current index', index, 'of', bucket.tweets.length); /* Reached the end of the current list of tweets in thread) */ - if (index >= (bucket.tweets.length - 1)) { + if (index >= bucket.tweets.length - 1) { /* See if we have a cursor to fetch more tweets */ - const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore')); - console.log('current cursors: ', bucket.cursors) + const cursor = bucket.cursors.find( + cursor => cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore' + ); + console.log('current cursors: ', bucket.cursors); if (!cursor) { - console.log('No cursor present, stopping pagination down') + console.log('No cursor present, stopping pagination down'); break; } console.log('Cursor present, fetching more tweets down'); - let loadCursor: GraphQLTweetFoundResponse; + let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value) + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); - if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + if ( + typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === + 'undefined' + ) { console.log('Unknown data while fetching cursor', loadCursor); break; } - } catch(e) { + } catch (e) { console.log('Error fetching cursor', e); break; } - const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); - bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet)); + const cursorResponse = processResponse( + loadCursor.data.threaded_conversation_with_injections_v2.instructions + ); + bucket.tweets = bucket.tweets.concat( + filterBucketTweets(cursorResponse.tweets, originalTweet) + ); /* Remove old cursor and add new bottom cursor if necessary */ consolidateCursors(bucket.cursors, cursorResponse.cursors); console.log('updated bucket of cursors', bucket.cursors); @@ -351,36 +427,54 @@ export const constructTwitterThread = async (id: string, const tweet = bucket.tweets[index]; const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; - console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + console.log( + 'adding previous tweet to thread', + newCurrentId, + 'from', + currentId, + 'at index', + index, + 'in bucket' + ); threadTweets.unshift(tweet); currentId = newCurrentId; - + if (index === 0) { /* See if we have a cursor to fetch more tweets */ - const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore')); - console.log('current cursors: ', bucket.cursors) + const cursor = bucket.cursors.find( + cursor => cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore' + ); + console.log('current cursors: ', bucket.cursors); if (!cursor) { - console.log('No cursor present, stopping pagination up') + console.log('No cursor present, stopping pagination up'); break; } console.log('Cursor present, fetching more tweets up'); - let loadCursor: GraphQLTweetFoundResponse; + + let loadCursor: TweetDetailResult; try { - loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value) + loadCursor = await fetchTweetDetail(id, request.event, true, cursor.value); - if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + if ( + typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === + 'undefined' + ) { console.log('Unknown data while fetching cursor', loadCursor); break; } - } catch(e) { + } catch (e) { console.log('Error fetching cursor', e); break; } - const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); - bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet)); + const cursorResponse = processResponse( + loadCursor.data.threaded_conversation_with_injections_v2.instructions + ); + bucket.tweets = cursorResponse.tweets.concat( + filterBucketTweets(bucket.tweets, originalTweet) + ); /* Remove old cursor and add new top cursor if necessary */ consolidateCursors(bucket.cursors, cursorResponse.cursors); @@ -396,14 +490,14 @@ export const constructTwitterThread = async (id: string, thread: [], author: author, code: 200 - } + }; - threadTweets.forEach(async (tweet) => { - socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false) as APITweet); + threadTweets.forEach(async tweet => { + socialThread.thread?.push((await buildAPITweet(tweet, undefined, true, false)) as APITweet); }); return socialThread; -} +}; export const threadAPIProvider = async (request: IRequest) => { const { id } = request.params; @@ -412,5 +506,5 @@ export const threadAPIProvider = async (request: IRequest) => { return new Response(JSON.stringify(processedResponse), { headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } - }) -} \ No newline at end of file + }); +}; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index e7b39be..ff46146 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -14,7 +14,7 @@ export const buildAPITweet = async ( threadPiece = false, legacyAPI = false // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { const apiTweet = {} as APITweet; /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. @@ -38,7 +38,11 @@ export const buildAPITweet = async ( if (typeof tweet.core === 'undefined') { console.log('Tweet still not valid', tweet); - return null; + if (tweet.__typename === 'TweetUnavailable' && tweet.reason === 'Protected') { + return { status: 401 }; + } else { + return { status: 404 }; + } } const graphQLUser = tweet.core.user_results.result; @@ -50,7 +54,9 @@ export const buildAPITweet = async ( /* Populating a lot of the basics */ apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; apiTweet.id = id; - apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); + apiTweet.text = unescapeText( + linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '') + ); if (!threadPiece) { apiTweet.author = { id: apiUser.id, @@ -75,7 +81,7 @@ export const buildAPITweet = async ( if (legacyAPI) { // @ts-expect-error Use retweets for legacy API apiTweet.retweets = tweet.legacy.retweet_count; - + // @ts-expect-error `tweets` is only part of legacy API apiTweet.author.tweets = apiTweet.author.posts; // @ts-expect-error Part of legacy API that we no longer are able to track @@ -139,11 +145,16 @@ export const buildAPITweet = async ( } apiTweet.media = {}; - + /* We found a quote tweet, let's process that too */ const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { - apiTweet.quote = (await buildAPITweet(quoteTweet, language, threadPiece, legacyAPI)) as APITweet; + apiTweet.quote = (await buildAPITweet( + quoteTweet, + language, + threadPiece, + legacyAPI + )) as APITweet; /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ if (apiTweet.embed_card === 'tweet' && apiTweet.quote !== null) { apiTweet.embed_card = apiTweet.quote.embed_card; @@ -211,7 +222,11 @@ export const buildAPITweet = async ( } } - if (apiTweet.media?.videos && apiTweet.media?.videos.length > 0 && apiTweet.embed_card !== 'player') { + if ( + apiTweet.media?.videos && + apiTweet.media?.videos.length > 0 && + apiTweet.embed_card !== 'player' + ) { apiTweet.embed_card = 'player'; } @@ -221,7 +236,9 @@ export const buildAPITweet = async ( const translateAPI = await translateTweet(tweet, '', language); if (translateAPI !== null && translateAPI?.translation) { apiTweet.translation = { - text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + text: unescapeText( + linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '') + ), source_lang: translateAPI?.sourceLanguage || '', target_lang: translateAPI?.destinationLanguage || '', source_lang_en: translateAPI?.localizedSourceLanguage || '' @@ -233,7 +250,7 @@ export const buildAPITweet = async ( // @ts-expect-error Use twitter_card for legacy API apiTweet.twitter_card = apiTweet.embed_card; // @ts-expect-error Part of legacy API that we no longer are able to track - apiTweet.color = null + apiTweet.color = null; // @ts-expect-error Use twitter_card for legacy API delete apiTweet.embed_card; if ((apiTweet.media.all?.length ?? 0) < 1 && !apiTweet.media.external) { @@ -243,4 +260,4 @@ export const buildAPITweet = async ( } return apiTweet; -}; \ No newline at end of file +}; diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 5df0992..e8e693d 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -355,6 +355,7 @@ type GraphQLTweet = { // Workaround result: GraphQLTweet; __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; + reason: string; // used for errors rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -451,14 +452,14 @@ type GraphQLTimelineTweet = { tweet_results: { result: GraphQLTweet | TweetTombstone; }; -} +}; type GraphQLTimelineCursor = { cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; itemType: 'TimelineTimelineCursor'; value: string; __typename: 'TimelineTimelineCursor'; -} +}; interface GraphQLBaseTimeline { entryType: string; @@ -469,16 +470,16 @@ type GraphQLTimelineItem = GraphQLBaseTimeline & { entryType: 'TimelineTimelineItem'; __typename: 'TimelineTimelineItem'; itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; -} +}; type GraphQLTimelineModule = GraphQLBaseTimeline & { entryType: 'TimelineTimelineModule'; __typename: 'TimelineTimelineModule'; items: { entryId: `conversationthread-${number}-tweet-${number}`; - item: GraphQLTimelineItem + item: GraphQLTimelineItem; }[]; -} +}; type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ @@ -501,7 +502,10 @@ type GraphQLConversationThread = { type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; +type ThreadInstruction = + | TimelineAddEntriesInstruction + | TimelineTerminateTimelineInstruction + | TimelineAddModulesInstruction; type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; @@ -543,7 +547,7 @@ type GraphQLTweetNotFoundResponse = { ]; data: Record; }; -type GraphQLTweetFoundResponse = { +type TweetDetailResult = { errors?: unknown[]; data: { threaded_conversation_with_injections_v2: { @@ -564,8 +568,7 @@ type TweetResultsByRestIdResult = { type TweetStub = { __typename: 'TweetUnavailable'; reason: 'NsfwLoggedOut' | 'Protected'; -} - +}; interface GraphQLProcessBucket { tweets: GraphQLTweet[]; diff --git a/src/types/types.d.ts b/src/types/types.d.ts index c0fde4b..b3465e6 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -144,7 +144,7 @@ interface APIPost { replying_to: { screen_name: string | null; post: string | null; - } | null + } | null; source: string | null; @@ -197,4 +197,8 @@ interface SocialThread { thread: (APIPost | APITweet)[] | null; author: APIUser | null; code: number; -} \ No newline at end of file +} + +interface FetchResults { + status: number; +} diff --git a/src/worker.ts b/src/worker.ts index 2a6e227..8acd6c1 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,7 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; -import { threadAPIProvider } from './providers/twitter/conversation'; +// import { threadAPIProvider } from './providers/twitter/conversation'; declare const globalThis: { fetchCompletedTime: number; @@ -454,7 +454,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); -router.get('/v2/twitter/thread/:id', threadAPIProvider) +// router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses) From edeb38fcead03571b0bcf8b5e1306d1640ea3106 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 04:55:47 -0400 Subject: [PATCH 11/16] Don't include global_screen_name in API yet --- src/api/user.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api/user.ts b/src/api/user.ts index dadc83c..cf3aab7 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -89,6 +89,9 @@ export const userAPI = async ( const response: UserAPIResponse = { code: 200, message: 'OK' } as UserAPIResponse; const apiUser: APIUser = (await populateUserProperties(userResponse)) as APIUser; + /* Currently, we haven't rolled this out as it's part of the proto-v2 API */ + delete apiUser.global_screen_name; + /* Finally, staple the User to the response and return it */ response.user = apiUser; From 54dc7972474e048d30c249d4cc4ae5f9ad5f12e7 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 04:56:02 -0400 Subject: [PATCH 12/16] Bugfix --- src/render/instantview.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/render/instantview.ts b/src/render/instantview.ts index 3b92d3e..8779695 100644 --- a/src/render/instantview.ts +++ b/src/render/instantview.ts @@ -156,7 +156,7 @@ const generateTweetFooter = (tweet: APIPost, isQuote = false): string => { joined: author.joined ? `📆 ${formatDate(new Date(author.joined))}` : '', following: truncateSocialCount(author.following), followers: truncateSocialCount(author.followers), - tweets: truncateSocialCount(author.tweets) + tweets: truncateSocialCount(author.posts) }) }); }; From 0384e5746ed486213578ec6665af0aca9202ae05 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 05:03:37 -0400 Subject: [PATCH 13/16] Translation workaround --- src/embed/status.ts | 2 +- src/providers/twitter/conversation.ts | 9 ++++++--- src/providers/twitter/processor.ts | 2 ++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/embed/status.ts b/src/embed/status.ts index 8922b20..7fae148 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -45,7 +45,7 @@ export const handleStatus = async ( status, fetchWithThreads, request, - undefined, + language, flags?.api ?? false ); diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 3fa8956..87a9dfa 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -272,13 +272,16 @@ export const constructTwitterThread = async ( language: string | undefined, legacyAPI = false ): Promise => { - console.log('legacyAPI', legacyAPI); + console.log('language', language); let response: TweetDetailResult | TweetResultsByRestIdResult | null = null; let post: APITweet; /* We can use TweetDetail on elongator accounts to increase per-account rate limit. - We also use TweetDetail to process threads (WIP) */ - if (typeof TwitterProxy !== 'undefined' && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { + We also use TweetDetail to process threads (WIP) + + Also - dirty hack. Right now, TweetDetails requests aren't working with language and I haven't figured out why. + I'll figure out why eventually, but for now just don't use TweetDetail for this. */ + if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { console.log('Using TweetDetail for request...'); response = (await fetchTweetDetail(id, request.event)) as TweetDetailResult; diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index ff46146..c11b436 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -230,6 +230,8 @@ export const buildAPITweet = async ( apiTweet.embed_card = 'player'; } + console.log('language?', language) + /* If a language is specified in API or by user, let's try translating it! */ if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { console.log(`Attempting to translate Tweet to ${language}...`); From eeaa86681e26ae338ce66784a1ce79d6e88d48af Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 05:04:57 -0400 Subject: [PATCH 14/16] Restore final value for TWEET_DETAIL_API --- src/experiments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/experiments.ts b/src/experiments.ts index 0021d51..39640bc 100644 --- a/src/experiments.ts +++ b/src/experiments.ts @@ -24,7 +24,7 @@ const Experiments: { [key in Experiment]: ExperimentConfig } = { [Experiment.TWEET_DETAIL_API]: { name: 'Tweet detail API', description: 'Use Tweet Detail API (where available with elongator)', - percentage: 0.5 + percentage: 0.75 } }; From bd62987b3630adecf4608c90fb2ec4a12934205b Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 05:07:01 -0400 Subject: [PATCH 15/16] typo --- src/providers/twitter/conversation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/twitter/conversation.ts b/src/providers/twitter/conversation.ts index 87a9dfa..20c0e3e 100644 --- a/src/providers/twitter/conversation.ts +++ b/src/providers/twitter/conversation.ts @@ -279,7 +279,7 @@ export const constructTwitterThread = async ( /* We can use TweetDetail on elongator accounts to increase per-account rate limit. We also use TweetDetail to process threads (WIP) - Also - dirty hack. Right now, TweetDetails requests aren't working with language and I haven't figured out why. + Also - dirty hack. Right now, TweetDetail requests aren't working with language and I haven't figured out why. I'll figure out why eventually, but for now just don't use TweetDetail for this. */ if (typeof TwitterProxy !== 'undefined' && !language && (experimentCheck(Experiment.TWEET_DETAIL_API) || processThread)) { console.log('Using TweetDetail for request...'); From 8f57f389ef313d3a2178756eba21bad07767b4d3 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 2 Nov 2023 05:22:09 -0400 Subject: [PATCH 16/16] Fix tests --- src/embed/status.ts | 16 +++++++++------- src/worker.ts | 5 ++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/embed/status.ts b/src/embed/status.ts index 7fae148..1a58182 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -25,19 +25,21 @@ export const returnError = (error: string): StatusResponse => { Like Twitter, we use the terminologies interchangably. */ export const handleStatus = async ( status: string, - mediaNumber?: number, - userAgent?: string, - flags?: InputFlags, - language?: string, - event?: FetchEvent + mediaNumber: number | undefined, + userAgent: string, + flags: InputFlags, + language: string, + event: FetchEvent, + request: IRequest // eslint-disable-next-line sonarjs/cognitive-complexity ): Promise => { console.log('Direct?', flags?.direct); - const request = (event as FetchEvent).request as IRequest; + console.log('event', event) + let fetchWithThreads = false; - if (request.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) { + if (request?.headers.get('user-agent')?.includes('Telegram') && !flags?.direct) { fetchWithThreads = true; } diff --git a/src/worker.ts b/src/worker.ts index 8acd6c1..c907606 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -132,6 +132,8 @@ const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputF console.log('Bypass bot check'); } + console.log('event', event) + /* This throws the necessary data to handleStatus (in status.ts) */ const statusResponse = await handleStatus( id?.match(/\d{2,20}/)?.[0] || '0', @@ -139,7 +141,8 @@ const statusRequest = async (request: IRequest, event: FetchEvent, flags: InputF userAgent, flags, language, - event + event, + request ); /* Complete responses are normally sent just by errors. Normal embeds send a `text` value. */