From 639fd7851366a68e5417258c9ebc95acb6c65c00 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 17 Oct 2023 04:30:35 -0400 Subject: [PATCH 1/4] Bump fakeChromeVersion --- src/helpers/useragent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/useragent.ts b/src/helpers/useragent.ts index 512d9bb..d09d536 100644 --- a/src/helpers/useragent.ts +++ b/src/helpers/useragent.ts @@ -1,6 +1,6 @@ /* We keep this value up-to-date for making our requests to Twitter as indistinguishable from normal user traffic as possible. */ -const fakeChromeVersion = 116; +const fakeChromeVersion = 118; const platformWindows = 'Windows NT 10.0; Win64; x64'; const platformMac = 'Macintosh; Intel Mac OS X 10_15_7'; const platformLinux = 'X11; Linux x86_64'; From 63f1e9d9ef79d06eb905a23d32c12928e9f101a3 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 17 Oct 2023 21:53:35 -0400 Subject: [PATCH 2/4] Update package-lock.json --- package-lock.json | 172 +++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ae70e6..007aff5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -716,9 +716,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20231010.0.tgz", - "integrity": "sha512-LM9ePAh88EGoQkYisAfdLMEDzcaMinRer0mY11GOiN4A9ZU+6APRVvhh5JBRzI0F6Dkb8nHtrzhisioWCRaY1w==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20231016.0.tgz", + "integrity": "sha512-rPAnF8Q25+eHEsAopihWeftPW/P0QapY9d7qaUmtOXztWdd6YPQ7JuiWVj4Nvjphge1BleehxAbo4I3Z4L2H1g==", "cpu": [ "x64" ], @@ -732,9 +732,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20231010.0.tgz", - "integrity": "sha512-Vr7Z1O+vJRCnVeWaF0YSv0EMHiMRY7yYCxr7O509FzvJAXsZuXZ7DYC5TAD7a8HSeeqsxFTAbF9jg0y9A2wKVw==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20231016.0.tgz", + "integrity": "sha512-MvydDdiLXt+jy57vrVZ2lU6EQwCdpieyZoN8uBXSWzfG3zR/6dxU1+okvPQPlHN0jtlufqPeHrpJyAqqgLHUKA==", "cpu": [ "arm64" ], @@ -748,9 +748,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20231010.0.tgz", - "integrity": "sha512-l9oDVPVhPEOHr1JpcGnLSsIf1h8sZnvcIC2Tl1zt+3p/KGFyGqGyAZJMLUoMJ54Q07oRE1x3KAu+JcWWEvdxpg==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20231016.0.tgz", + "integrity": "sha512-y6Sj37yTzM8QbAghG9LRqoSBrsREnQz8NkcmpjSxeK6KMc2g0L5A/OemCdugNlIiv+zRv9BYX1aosaoxY5JbeQ==", "cpu": [ "x64" ], @@ -764,9 +764,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20231010.0.tgz", - "integrity": "sha512-NBmYsJu+ns2W8WHcDnglfqLV5O3FP7lXpoTSTvpM64mhexmemdMlOJX5gpRuarTula3fA+GzEehinUojwM9/1g==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20231016.0.tgz", + "integrity": "sha512-LqMIRUHD1YeRg2TPIfIQEhapSKMFSq561RypvJoXZvTwSbaROxGdW6Ku+PvButqTkEvuAtfzN/kGje7fvfQMHg==", "cpu": [ "arm64" ], @@ -780,9 +780,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20231010.0.tgz", - "integrity": "sha512-jWiG71Rvuh4FYdEpOP1+BAygdguTlMYYy+v5d4ZOjxDkl+V8aR86EEtDQrv/QLUJFbpcoEX25SxXnN5UMKtjhQ==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20231016.0.tgz", + "integrity": "sha512-96ojBwIHyiUAbsWlzBqo9P/cvH8xUh8SuBboFXtwAeXcJ6/urwKN2AqPa/QzOGUTCdsurWYiieARHT5WWWPhKw==", "cpu": [ "x64" ], @@ -796,9 +796,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "4.20231010.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20231010.0.tgz", - "integrity": "sha512-Jscqg52ScJMTMRHHVeoUvXYYiXPT4V59NKaClQVju9B6oPAzuePFYHwi5PLQ5Yr+xEbJ6+bHuq1aYV/69z6L0w==", + "version": "4.20231016.0", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20231016.0.tgz", + "integrity": "sha512-eGB0cRVyoJpeyGJx2re5sbd9R316a61sY73xwnqm4cwGpb+OxCK2gc651RxGiN7H4w6LY1RpysUgeGLmj5B3+g==", "dev": true }, "node_modules/@esbuild-plugins/node-globals-polyfill": { @@ -1735,9 +1735,9 @@ "dev": true }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.19", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz", - "integrity": "sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==", + "version": "0.3.20", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.20.tgz", + "integrity": "sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==", "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", @@ -2038,14 +2038,14 @@ } }, "node_modules/@sentry-internal/tracing": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.0.tgz", - "integrity": "sha512-JK6IRGgdtZjswGfaGIHNWIThffhOHzVIIaGmglui+VFIzOsOqePjoxaDV0MEvzafxXZD7eWqGE5RGuZ0n6HFVg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry-internal/tracing/-/tracing-7.74.1.tgz", + "integrity": "sha512-nNaiZreQxCitG2PzYPaC7XtyA9OMsETGYMKAtiK4p62/uTmeYbsBva9BoNx1XeiHRwbrVQYRMKQ9nV5e2jS4/A==", "dev": true, "dependencies": { - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2092,13 +2092,13 @@ } }, "node_modules/@sentry/core": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.0.tgz", - "integrity": "sha512-83NRuqn7nDZkSVBN5yJQqcpXDG4yMYiB7TkYUKrGTzBpRy6KUOrkCdybuKk0oraTIGiGSe5WEwCFySiNgR9FzA==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-7.74.1.tgz", + "integrity": "sha512-LvEhOSfdIvwkr+PdlrT/aA/iOLhkXrSkvjqAQyogE4ddCWeYfS0NoirxNt1EaxMBAWKhYZRqzkA7WA4LDLbzlA==", "dev": true, "dependencies": { - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2120,14 +2120,14 @@ } }, "node_modules/@sentry/integrations": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.74.0.tgz", - "integrity": "sha512-O4UyxiV5wzXSDnEd9Z/SIt/5M12URWNtIJPPJjowlllzw8X9e3zBcnXmjMOLZ+mZWjQmRDjOoz3lPPQ17f7fvw==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/integrations/-/integrations-7.74.1.tgz", + "integrity": "sha512-Q7chPehHpHB4WOQ1J/X6NiN2ptiqJMmxtL+6wHumzIAyrjup3c9XekR83qEs8zpqYJAlb/4MUlwd9fPbkhGXnQ==", "dev": true, "dependencies": { - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "localforage": "^1.8.1", "tslib": "^2.4.1 || ^1.9.3" }, @@ -2136,15 +2136,15 @@ } }, "node_modules/@sentry/node": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.0.tgz", - "integrity": "sha512-uBmW2/z0cz/WFIG74ZF7lSipO0XNzMf9yrdqnZXnGDYsUZE4I4QiqDN0hNi6fkTgf9MYRC8uFem2OkAvyPJ74Q==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-7.74.1.tgz", + "integrity": "sha512-aMUQ2LFZF64FBr+cgjAqjT4OkpYBIC9lyWI8QqjEHqNho5+LGu18/iVrJPD4fgs4UhGdCuAiQjpC36MbmnIDZA==", "dev": true, "dependencies": { - "@sentry-internal/tracing": "7.74.0", - "@sentry/core": "7.74.0", - "@sentry/types": "7.74.0", - "@sentry/utils": "7.74.0", + "@sentry-internal/tracing": "7.74.1", + "@sentry/core": "7.74.1", + "@sentry/types": "7.74.1", + "@sentry/utils": "7.74.1", "cookie": "^0.5.0", "https-proxy-agent": "^5.0.0", "lru_map": "^0.3.3", @@ -2155,21 +2155,21 @@ } }, "node_modules/@sentry/types": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.0.tgz", - "integrity": "sha512-rI5eIRbUycWjn6s6o3yAjjWtIvYSxZDdnKv5je2EZINfLKcMPj1dkl6wQd2F4y7gLfD/N6Y0wZYIXC3DUdJQQg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-7.74.1.tgz", + "integrity": "sha512-2jIuPc+YKvXqZETwr2E8VYnsH1zsSUR/wkIvg1uTVeVNyoowJv+YsOtCdeGyL2AwiotUBSPKu7O1Lz0kq5rMOQ==", "dev": true, "engines": { "node": ">=8" } }, "node_modules/@sentry/utils": { - "version": "7.74.0", - "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.0.tgz", - "integrity": "sha512-k3np8nuTPtx5KDODPtULfFln4UXdE56MZCcF19Jv6Ljxf+YN/Ady1+0Oi3e0XoSvFpWNyWnglauT7M65qCE6kg==", + "version": "7.74.1", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-7.74.1.tgz", + "integrity": "sha512-qUsqufuHYcy5gFhLZslLxA5kcEOkkODITXW3c7D+x+8iP/AJqa8v8CeUCVNS7RetHCuIeWAbbTClC4c411EwQg==", "dev": true, "dependencies": { - "@sentry/types": "7.74.0", + "@sentry/types": "7.74.1", "tslib": "^2.4.1 || ^1.9.3" }, "engines": { @@ -2201,9 +2201,9 @@ } }, "node_modules/@types/babel__core": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.2.tgz", - "integrity": "sha512-pNpr1T1xLUc2l3xJKuPtsEky3ybxN3m4fJkknfIpTCTfIZCDW57oAg+EfCgIIp2rvCe0Wn++/FfodDS4YXxBwA==", + "version": "7.20.3", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.3.tgz", + "integrity": "sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA==", "dev": true, "dependencies": { "@babel/parser": "^7.20.7", @@ -2214,18 +2214,18 @@ } }, "node_modules/@types/babel__generator": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.5.tgz", - "integrity": "sha512-h9yIuWbJKdOPLJTbmSpPzkF67e659PbQDba7ifWm5BJ8xTv+sDmS7rFmywkWOvXedGTivCdeGSIIX8WLcRTz8w==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.6.tgz", + "integrity": "sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w==", "dev": true, "dependencies": { "@babel/types": "^7.0.0" } }, "node_modules/@types/babel__template": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.2.tgz", - "integrity": "sha512-/AVzPICMhMOMYoSx9MoKpGDKdBRsIXMNByh1PXSZoa+v6ZoLa8xxtsT/uLQ/NJm0XVAWl/BvId4MlDeXJaeIZQ==", + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.3.tgz", + "integrity": "sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ==", "dev": true, "dependencies": { "@babel/parser": "^7.1.0", @@ -2233,18 +2233,18 @@ } }, "node_modules/@types/babel__traverse": { - "version": "7.20.2", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.2.tgz", - "integrity": "sha512-ojlGK1Hsfce93J0+kn3H5R73elidKUaZonirN33GSmgTUMpzI/MIFfSpF3haANe3G1bEBS9/9/QEqwTzwqFsKw==", + "version": "7.20.3", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.3.tgz", + "integrity": "sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw==", "dev": true, "dependencies": { "@babel/types": "^7.20.7" } }, "node_modules/@types/better-sqlite3": { - "version": "7.6.5", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.5.tgz", - "integrity": "sha512-H3ZUx89KiPhYa9nalUXVVStSUFHuzYxt4yoazufpTTYW9rVUCzhh02V8CH2C8nE4libnK0UgFq5DFIe0DOhqow==", + "version": "7.6.6", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.6.tgz", + "integrity": "sha512-nuFAptzt0hZYBvyLzKQCbuCCK+RN9PHH4ezar5EJLIg2qpVhwQ/uLvLO/K8A9O7N8DafawgFupiyXQSs0U48Ng==", "dev": true, "dependencies": { "@types/node": "*" @@ -3494,9 +3494,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.554", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.554.tgz", - "integrity": "sha512-Q0umzPJjfBrrj8unkONTgbKQXzXRrH7sVV7D9ea2yBV3Oaogz991yhbpfvo2LMNkJItmruXTEzVpP9cp7vaIiQ==", + "version": "1.4.557", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.557.tgz", + "integrity": "sha512-6x0zsxyMXpnMJnHrondrD3SuAeKcwij9S+83j2qHAQPXbGTDDfgImzzwgGlzrIcXbHQ42tkG4qA6U860cImNhw==", "dev": true }, "node_modules/emittery": { @@ -5418,9 +5418,9 @@ } }, "node_modules/miniflare": { - "version": "3.20231010.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20231010.0.tgz", - "integrity": "sha512-VETY+/OhJ1RN+yrFpPUqBZysb2R8wXvyx3vzaRZS2qO1aGNKeGASa/vxCvNcBF+gt8UdbWMOalSXX8zY0IgWZA==", + "version": "3.20231016.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-3.20231016.0.tgz", + "integrity": "sha512-AmlqI89zsnBJfC+nKKZdCB/fuu0q/br24Kqt9NZwcT6yJEpO5NytNKfjl6nJROHROwuJSRQR1T3yopCtG1/0DA==", "dev": true, "dependencies": { "acorn": "^8.8.0", @@ -5431,7 +5431,7 @@ "source-map-support": "0.5.21", "stoppable": "^1.1.0", "undici": "^5.22.1", - "workerd": "1.20231010.0", + "workerd": "1.20231016.0", "ws": "^8.11.0", "youch": "^3.2.2", "zod": "^3.20.6" @@ -7300,9 +7300,9 @@ } }, "node_modules/workerd": { - "version": "1.20231010.0", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20231010.0.tgz", - "integrity": "sha512-ghxfBU8fBSBDa8fCBPfzWivYsWpewYftgy70N308C+acQ5AaKNM1QTdkQNm9YWeC5Jpl1YvBX04ojt7lCc3juw==", + "version": "1.20231016.0", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20231016.0.tgz", + "integrity": "sha512-v2GDb5XitSqgub/xm7EWHVAlAK4snxQu3itdMQxXstGtUG9hl79fQbXS/8fNFbmms2R2bAxUwSv47q8k5T5Erw==", "dev": true, "hasInstallScript": true, "bin": { @@ -7312,17 +7312,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20231010.0", - "@cloudflare/workerd-darwin-arm64": "1.20231010.0", - "@cloudflare/workerd-linux-64": "1.20231010.0", - "@cloudflare/workerd-linux-arm64": "1.20231010.0", - "@cloudflare/workerd-windows-64": "1.20231010.0" + "@cloudflare/workerd-darwin-64": "1.20231016.0", + "@cloudflare/workerd-darwin-arm64": "1.20231016.0", + "@cloudflare/workerd-linux-64": "1.20231016.0", + "@cloudflare/workerd-linux-arm64": "1.20231016.0", + "@cloudflare/workerd-windows-64": "1.20231016.0" } }, "node_modules/wrangler": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.13.1.tgz", - "integrity": "sha512-CY73h4lfPx/3CmkC/tPj66DRRZ9Y42sMcHys6B6tjCILUo950IeOvnsj759el3/ewFLY4kG4jCrrrikan6TE+Q==", + "version": "3.13.2", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-3.13.2.tgz", + "integrity": "sha512-Z/ZrAL2mJc7E4ialOV9c3wGry0qagp9DfPl5XVd67vtlFPeTSR+1pemtZ5+qI2BXi59kP3OGHBKrrIyG0d9csg==", "dev": true, "dependencies": { "@cloudflare/kv-asset-handler": "^0.2.0", @@ -7331,7 +7331,7 @@ "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", "esbuild": "0.17.19", - "miniflare": "3.20231010.0", + "miniflare": "3.20231016.0", "nanoid": "^3.3.3", "path-to-regexp": "^6.2.0", "selfsigned": "^2.0.1", From 0a616e973178b57be751d4fbb9867e7b4a2e9207 Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Tue, 17 Oct 2023 23:20:46 -0400 Subject: [PATCH 3/4] Add twitter thread api (WIP) --- src/api/status.ts | 20 +- src/api/user.ts | 8 +- src/constants.ts | 1 + src/embed/status.ts | 12 +- src/providers/twitter/processor.ts | 211 ++++++++++++++++++++ src/providers/twitter/status.ts | 308 +++++++++++++++++++++++++++++ src/types/twitterTypes.d.ts | 69 +++++-- src/types/types.d.ts | 51 ++--- src/worker.ts | 2 + 9 files changed, 627 insertions(+), 55 deletions(-) create mode 100644 src/providers/twitter/processor.ts create mode 100644 src/providers/twitter/status.ts diff --git a/src/api/status.ts b/src/api/status.ts index 76a6e8c..ef129c7 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -56,6 +56,7 @@ const populateTweetProperties = async ( name: apiUser.name, screen_name: apiUser.screen_name, avatar_url: (apiUser.avatar_url || '').replace('_normal', '_200x200') || '', + // @ts-expect-error Legacy api shit avatar_color: null, banner_url: apiUser.banner_url || '', description: apiUser.description || '', @@ -72,8 +73,11 @@ const populateTweetProperties = async ( }; apiTweet.replies = tweet.legacy.reply_count; apiTweet.retweets = tweet.legacy.retweet_count; + apiTweet.reposts = tweet.legacy.retweet_count; apiTweet.likes = tweet.legacy.favorite_count; + // @ts-expect-error Legacy api shit apiTweet.color = null; + // @ts-expect-error legacy api apiTweet.twitter_card = 'tweet'; apiTweet.created_at = tweet.legacy.created_at; apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; @@ -126,11 +130,11 @@ const populateTweetProperties = async ( apiTweet.media.all.push(mediaObject); if (mediaObject.type === 'photo') { - apiTweet.twitter_card = 'summary_large_image'; + apiTweet.embed_card = 'summary_large_image'; apiTweet.media.photos = apiTweet.media.photos || []; apiTweet.media.photos.push(mediaObject); } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { - apiTweet.twitter_card = 'player'; + apiTweet.embed_card = 'player'; apiTweet.media.videos = apiTweet.media.videos || []; apiTweet.media.videos.push(mediaObject); } else { @@ -178,9 +182,9 @@ const populateTweetProperties = async ( /* Workaround: Force player card by default for videos */ /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' The mediaList however can set it to something else. TODO: Reimplement as enums */ - // @ts-expect-error see above comment - if (apiTweet.media?.videos && apiTweet.twitter_card !== 'player') { - apiTweet.twitter_card = 'player'; + + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; } /* If a language is specified in API or by user, let's try translating it! */ @@ -291,9 +295,9 @@ export const statusAPI = async ( const quoteTweet = tweet.quoted_status_result; if (quoteTweet) { apiTweet.quote = (await populateTweetProperties(quoteTweet, res, language)) as APITweet; - /* Only override the twitter_card if it's a basic tweet, since media always takes precedence */ - if (apiTweet.twitter_card === 'tweet') { - apiTweet.twitter_card = apiTweet.quote.twitter_card; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet') { + apiTweet.embed_card = apiTweet.quote.embed_card; } } diff --git a/src/api/user.ts b/src/api/user.ts index 45885eb..499900c 100644 --- a/src/api/user.ts +++ b/src/api/user.ts @@ -11,11 +11,13 @@ export const convertToApiUser = (user: GraphQLUser): APIUser => { apiUser.following = user.legacy.friends_count; apiUser.likes = user.legacy.favourites_count; apiUser.tweets = user.legacy.statuses_count; + apiUser.posts = user.legacy.statuses_count; apiUser.name = user.legacy.name; apiUser.screen_name = user.legacy.screen_name; - apiUser.description = linkFixer(user.legacy.entities?.description?.urls, user.legacy.description); - apiUser.location = user.legacy.location; - apiUser.banner_url = user.legacy.profile_banner_url; + apiUser.global_screen_name = `${user.legacy.screen_name}@${Constants.TWITTER_GLOBAL_NAME_ROOT}` + apiUser.description = user.legacy.description ? linkFixer(user.legacy.entities?.description?.urls, user.legacy.description) : null; + apiUser.location = user.legacy.location ? user.legacy.location : null; + apiUser.banner_url = user.legacy.profile_banner_url ? user.legacy.profile_banner_url : null; /* if (user.is_blue_verified) { apiUser.verified = 'blue'; diff --git a/src/constants.ts b/src/constants.ts index 5c8ff7e..addbee0 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,6 +15,7 @@ export const Constants = { RELEASE_NAME: RELEASE_NAME, API_DOCS_URL: `https://github.com/dangeredwolf/FixTweet/wiki/API-Home`, TWITTER_ROOT: 'https://twitter.com', + TWITTER_GLOBAL_NAME_ROOT: 'twitter.com', TWITTER_API_ROOT: 'https://api.twitter.com', BOT_UA_REGEX: /bot|facebook|embed|got|firefox\/92|firefox\/38|curl|wget|go-http|yahoo|generator|whatsapp|preview|link|proxy|vkshare|images|analyzer|index|crawl|spider|python|cfnetwork|node|mastodon|http\.rb/gi, diff --git a/src/embed/status.ts b/src/embed/status.ts index 1bab6d8..7f7df01 100644 --- a/src/embed/status.ts +++ b/src/embed/status.ts @@ -120,7 +120,7 @@ export const handleStatus = async ( const headers = [ ``, ``, - ``, + ``, ``, ``, `` @@ -199,8 +199,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card === 'player') { - tweet.twitter_card = 'summary_large_image'; + if (tweet.embed_card === 'player') { + tweet.embed_card = 'summary_large_image'; } break; case 'video': @@ -216,8 +216,8 @@ export const handleStatus = async ( siteName = instructions.siteName; } /* Overwrite our Twitter Card if overriding media, so it displays correctly in Discord */ - if (tweet.twitter_card !== 'player') { - tweet.twitter_card = 'player'; + if (tweet.embed_card !== 'player') { + tweet.embed_card = 'player'; } /* This Tweet has a video to render. */ break; @@ -344,7 +344,7 @@ export const handleStatus = async ( and we have to pretend to be Medium in order to get working IV, but haven't figured if the template is causing issues. */ const text = useIV ? sanitizeText(newText).replace(/\n/g, '
') : sanitizeText(newText); - const useCard = tweet.twitter_card === 'tweet' ? tweet.quote?.twitter_card : tweet.twitter_card; + const useCard = tweet.embed_card === 'tweet' ? tweet.quote?.embed_card : tweet.embed_card; /* Push basic headers relating to author, Tweet text, and site name */ headers.push( diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts new file mode 100644 index 0000000..d28203b --- /dev/null +++ b/src/providers/twitter/processor.ts @@ -0,0 +1,211 @@ +import { renderCard } from '../../helpers/card'; +import { Constants } from '../../constants'; +import { linkFixer } from '../../helpers/linkFixer'; +import { handleMosaic } from '../../helpers/mosaic'; +// import { translateTweet } from '../../helpers/translate'; +import { unescapeText } from '../../helpers/utils'; +import { processMedia } from '../../helpers/media'; +import { convertToApiUser } from '../../api/user'; + +export const buildAPITweet = async ( + tweet: GraphQLTweet, + language: string | undefined, + threadPiece = false, + legacyAPI = false + // eslint-disable-next-line sonarjs/cognitive-complexity +): Promise => { + const apiTweet = {} as APITweet; + + /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. + It has slightly different attributes from the regular 'Tweet' type. We fix that up here. */ + + if (typeof tweet.core === 'undefined' && typeof tweet.result !== 'undefined') { + tweet = tweet.result; + } + + if (typeof tweet.core === 'undefined' && typeof tweet.tweet?.core !== 'undefined') { + tweet.core = tweet.tweet.core; + } + + if (typeof tweet.legacy === 'undefined' && typeof tweet.tweet?.legacy !== 'undefined') { + tweet.legacy = tweet.tweet?.legacy; + } + + if (typeof tweet.views === 'undefined' && typeof tweet?.tweet?.views !== 'undefined') { + tweet.views = tweet?.tweet?.views; + } + + const graphQLUser = tweet.core.user_results.result; + const apiUser = convertToApiUser(graphQLUser); + + /* Sometimes, `rest_id` is undefined for some reason. Inconsistent behavior. See: https://github.com/FixTweet/FixTweet/issues/416 */ + const id = tweet.rest_id ?? tweet.legacy.id_str; + + /* Populating a lot of the basics */ + apiTweet.url = `${Constants.TWITTER_ROOT}/${apiUser.screen_name}/status/${id}`; + apiTweet.id = id; + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities?.urls, tweet.legacy.full_text || '')); + if (!threadPiece) { + apiTweet.author = { + id: apiUser.id, + name: apiUser.name, + screen_name: apiUser.screen_name, + global_screen_name: apiUser.global_screen_name, + avatar_url: apiUser.avatar_url?.replace?.('_normal', '_200x200') ?? null, + banner_url: apiUser.banner_url ?? null, + description: apiUser.description ?? null, + location: apiUser.location ?? null, + url: apiUser.url ?? null, + followers: apiUser.followers, + following: apiUser.following, + joined: apiUser.joined, + posts: apiUser.tweets, + likes: apiUser.likes, + protected: apiUser.protected, + birthday: apiUser.birthday, + website: apiUser.website + }; + } + apiTweet.replies = tweet.legacy.reply_count; + if (legacyAPI) { + apiTweet.retweets = tweet.legacy.retweet_count; + } else { + apiTweet.reposts = tweet.legacy.retweet_count; + } + apiTweet.likes = tweet.legacy.favorite_count; + apiTweet.embed_card = 'tweet'; + apiTweet.created_at = tweet.legacy.created_at; + apiTweet.created_timestamp = new Date(tweet.legacy.created_at).getTime() / 1000; + + apiTweet.possibly_sensitive = tweet.legacy.possibly_sensitive; + + if (tweet.views.state === 'EnabledWithCount') { + apiTweet.views = parseInt(tweet.views.count || '0') ?? null; + } else { + apiTweet.views = null; + } + console.log('note_tweet', JSON.stringify(tweet.note_tweet)); + const noteTweetText = tweet.note_tweet?.note_tweet_results?.result?.text; + + if (noteTweetText) { + tweet.legacy.entities.urls = tweet.note_tweet?.note_tweet_results?.result?.entity_set.urls; + tweet.legacy.entities.hashtags = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.hashtags; + tweet.legacy.entities.symbols = + tweet.note_tweet?.note_tweet_results?.result?.entity_set.symbols; + + console.log('We meet the conditions to use new note tweets'); + apiTweet.text = unescapeText(linkFixer(tweet.legacy.entities.urls, noteTweetText)); + apiTweet.is_note_tweet = true; + } else { + apiTweet.is_note_tweet = false; + } + + if (tweet.legacy.lang !== 'unk') { + apiTweet.lang = tweet.legacy.lang; + } else { + apiTweet.lang = null; + } + + if (legacyAPI) { + apiTweet.replying_to = tweet.legacy?.in_reply_to_screen_name || null; + apiTweet.replying_to_status = tweet.legacy?.in_reply_to_status_id_str || null; + } else if (tweet.legacy.in_reply_to_screen_name) { + apiTweet.reply_of = { + screen_name: tweet.legacy.in_reply_to_screen_name || null, + post: tweet.legacy.in_reply_to_status_id_str || null + }; + } else { + apiTweet.reply_of = null; + } + + apiTweet.media = { + all: [], + photos: [], + videos: [], + }; + + const mediaList = Array.from( + tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || [] + ); + + // console.log('tweet', JSON.stringify(tweet)); + + /* Populate this Tweet's media */ + mediaList.forEach(media => { + const mediaObject = processMedia(media); + if (mediaObject) { + apiTweet.media?.all?.push(mediaObject); + if (mediaObject.type === 'photo') { + apiTweet.embed_card = 'summary_large_image'; + apiTweet.media?.photos?.push(mediaObject); + } else if (mediaObject.type === 'video' || mediaObject.type === 'gif') { + apiTweet.embed_card = 'player'; + apiTweet.media?.videos?.push(mediaObject); + } else { + console.log('Unknown media type', mediaObject.type); + } + } + }); + + /* Grab color palette data */ + /* + if (mediaList[0]?.ext_media_color?.palette) { + apiTweet.color = colorFromPalette(mediaList[0].ext_media_color.palette); + } + */ + + /* Handle photos and mosaic if available */ + if ((apiTweet?.media?.photos?.length || 0) > 1 && !threadPiece) { + const mosaic = await handleMosaic(apiTweet.media?.photos || [], id); + if (typeof apiTweet.media !== 'undefined' && mosaic !== null) { + apiTweet.media.mosaic = mosaic; + } + } + + // Add Tweet source but remove the link HTML tag + if (tweet.source) { + apiTweet.source = (tweet.source || '').replace( + /(.+?)<\/a>/, + '$2' + ); + } + + /* Populate a Twitter card */ + + if (tweet.card) { + const card = renderCard(tweet.card); + if (card.external_media) { + apiTweet.media = apiTweet.media ?? {}; + apiTweet.media.external = card.external_media; + } + if (card.poll) { + apiTweet.poll = card.poll; + } + } + + /* Workaround: Force player card by default for videos */ + /* TypeScript gets confused and re-interprets the type'tweet' instead of 'tweet' | 'summary' | 'summary_large_image' | 'player' + The mediaList however can set it to something else. TODO: Reimplement as enums */ + // @ts-expect-error see above comment + if (apiTweet.media?.videos && apiTweet.embed_card !== 'player') { + apiTweet.embed_card = 'player'; + } + + /* If a language is specified in API or by user, let's try translating it! */ + if (typeof language === 'string' && language.length === 2 && language !== tweet.legacy.lang) { + /* TODO: Reimplement */ + // console.log(`Attempting to translate Tweet to ${language}...`); + // const translateAPI = await translateTweet(tweet, conversation.guestToken || '', language); + // if (translateAPI !== null && translateAPI?.translation) { + // apiTweet.translation = { + // text: unescapeText(linkFixer(tweet.legacy?.entities?.urls, translateAPI?.translation || '')), + // source_lang: translateAPI?.sourceLanguage || '', + // target_lang: translateAPI?.destinationLanguage || '', + // source_lang_en: translateAPI?.localizedSourceLanguage || '' + // }; + // } + } + + return apiTweet; +}; \ No newline at end of file diff --git a/src/providers/twitter/status.ts b/src/providers/twitter/status.ts new file mode 100644 index 0000000..f43c141 --- /dev/null +++ b/src/providers/twitter/status.ts @@ -0,0 +1,308 @@ +import { IRequest } from "itty-router"; +import { Constants } from "../../constants"; +import { twitterFetch } from "../../fetch"; +import { buildAPITweet } from "./processor"; + +type GraphQLProcessBucket = { + tweets: GraphQLTweet[]; + cursors: GraphQLTimelineCursor[]; +} + +type SocialThread = { + post: APIPost | APITweet | null; + thread: (APIPost | APITweet)[] | null; + author: APIUser | null; +} + +export const fetchTwitterThread = async ( + status: string, + event: FetchEvent, + useElongator = typeof TwitterProxy !== 'undefined', + cursor: string | null = null +): Promise => { + return (await twitterFetch( + `${ + Constants.TWITTER_ROOT + }/i/api/graphql/7xdlmKfKUJQP7D7woCL5CA/TweetDetail?variables=${encodeURIComponent( + JSON.stringify({ + focalTweetId: status, + referrer: "home", + with_rux_injections: false, + includePromotedContent: false, + withCommunity: true, + withBirdwatchNotes: true, + withQuickPromoteEligibilityTweetFields: false, + withVoice: false, + withV2Timeline: true, + cursor: cursor + }) + )}&features=${encodeURIComponent( + JSON.stringify({ + responsive_web_graphql_exclude_directive_enabled: true, + verified_phone_label_enabled: false, + responsive_web_home_pinned_timelines_enabled: true, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + tweetypie_unmention_optimization_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: false, + tweet_awards_web_tipping_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_media_download_video_enabled: true, + responsive_web_enhance_cards_enabled: true + }) + )}&fieldToggles=${encodeURIComponent( + JSON.stringify({ + withArticleRichContentState: true + }) + )}`, + event, + useElongator, + () => { + return true; + } + )) as GraphQLTweetFoundResponse; +}; + +const processResponse = (instructions: V2ThreadInstruction[]): GraphQLProcessBucket => { + const bucket: GraphQLProcessBucket = { + tweets: [], + cursors: [] + }; + instructions.forEach?.(instruction => { + if (instruction.type === 'TimelineAddEntries' || instruction.type === 'TimelineAddToModule') { + // @ts-expect-error Use entries or moduleItems depending on the type + (instruction?.entries ?? instruction.moduleItems).forEach((_entry) => { + const entry = _entry as GraphQLTimelineTweetEntry | GraphQLConversationThread | GraphQLModuleTweetEntry + const content = (entry as GraphQLModuleTweetEntry)?.item ?? (entry as GraphQLTimelineTweetEntry)?.content; + if (content.__typename === 'TimelineTimelineItem') { + const itemContentType = content.itemContent?.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = content.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(content.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(content.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(content.itemContent as GraphQLTimelineCursor); + } + } else if ((content as unknown as GraphQLTimelineModule).__typename === 'TimelineTimelineModule') { + content.items.forEach((item) => { + const itemContentType = item.item.itemContent.__typename; + if (itemContentType === 'TimelineTweet') { + const entryType = item.item.itemContent.tweet_results.result.__typename + if (entryType === 'Tweet') { + bucket.tweets.push(item.item.itemContent.tweet_results.result as GraphQLTweet); + } + if (entryType === 'TweetWithVisibilityResults') { + bucket.tweets.push(item.item.itemContent.tweet_results.result.tweet as GraphQLTweet); + } + } else if (itemContentType === 'TimelineTimelineCursor') { + bucket.cursors.push(item.item.itemContent as GraphQLTimelineCursor); + } + }); + } + }); + } + }) + + return bucket; +} + +const findTweetInBucket = (id: string, bucket: GraphQLProcessBucket): GraphQLTweet | null => { + return bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id) ?? null; +} + +const findNextTweet = (id: string, bucket: GraphQLProcessBucket): number => { + return bucket.tweets.findIndex(tweet => tweet.legacy?.in_reply_to_status_id_str === id); +} + +const findPreviousTweet = (id: string, bucket: GraphQLProcessBucket): number => { + const tweet = bucket.tweets.find(tweet => (tweet.rest_id ?? tweet.legacy?.id_str) === id); + if (!tweet) { + console.log('uhhh, we could not even find that tweet, dunno how that happened'); + return -1; + } + const index = bucket.tweets.findIndex(_tweet => (_tweet.rest_id ?? _tweet.legacy?.id_str) === tweet.legacy?.in_reply_to_status_id_str); + if (index === -1) { + console.log('could not find shit for', id) + console.log(bucket.cursors) + } + return index; +} + +const consolidateCursors = (oldCursors: GraphQLTimelineCursor[], newCursors: GraphQLTimelineCursor[]): GraphQLTimelineCursor[] => { + /* Update the Bottom/Top cursor with the new one if applicable. Otherwise, keep the old one */ + return oldCursors.map(cursor => { + const newCursor = newCursors.find(_cursor => _cursor.cursorType === cursor.cursorType); + if (newCursor) { + return newCursor; + } + return cursor; + }); +} + +const filterBucketTweets = (tweets: GraphQLTweet[], original: GraphQLTweet) => { + return tweets.filter(tweet => tweet.core?.user_results?.result?.rest_id === original.core?.user_results?.result?.rest_id) +} + +export const processTwitterThread = async (id: string, processThread = false, request: IRequest): Promise => { + const response = await fetchTwitterThread(id, request.event) as GraphQLTweetFoundResponse; + + if (!response.data) { + return { post: null, thread: null, author: null }; + } + + const bucket = processResponse(response.data.threaded_conversation_with_injections_v2.instructions); + const originalTweet = findTweetInBucket(id, bucket); + + /* Don't bother processing thread on a null tweet */ + if (originalTweet === null) { + return { post: null, thread: null, author: null }; + } + + const post = await buildAPITweet(originalTweet, undefined, false, false); + const author = post.author; + /* remove post.author */ + // @ts-expect-error lmao + delete post.author; + + /* If we're not processing threads, let's be done here */ + if (!processThread) { + return { post: post, thread: null, author: author }; + } + + const threadTweets = [originalTweet]; + bucket.tweets = filterBucketTweets(bucket.tweets, originalTweet); + + let currentId = id; + + /* Process tweets that are following the current one in the thread */ + while (findNextTweet(currentId, bucket) !== -1) { + const index = findNextTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding next tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.push(tweet); + + currentId = newCurrentId; + + console.log('Current index', index, 'of', bucket.tweets.length) + + /* Reached the end of the current list of tweets in thread) */ + if (index >= (bucket.tweets.length - 1)) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Bottom' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination down') + break; + } + console.log('Cursor present, fetching more tweets down'); + + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = bucket.tweets.concat(filterBucketTweets(cursorResponse.tweets, originalTweet)); + /* Remove old cursor and add new bottom cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of next tweet:', findNextTweet(currentId, bucket)); + } + + currentId = id; + + while (findPreviousTweet(currentId, bucket) !== -1) { + const index = findPreviousTweet(currentId, bucket); + const tweet = bucket.tweets[index]; + const newCurrentId = tweet.rest_id ?? tweet.legacy?.id_str; + + console.log('adding previous tweet to thread', newCurrentId, 'from', currentId, 'at index', index, 'in bucket') + + threadTweets.unshift(tweet); + + currentId = newCurrentId; + + if (index === 0) { + /* See if we have a cursor to fetch more tweets */ + const cursor = bucket.cursors.find(cursor => (cursor.cursorType === 'Top' || cursor.cursorType === 'ShowMore')); + console.log('current cursors: ', bucket.cursors) + if (!cursor) { + console.log('No cursor present, stopping pagination up') + break; + } + console.log('Cursor present, fetching more tweets up'); + let loadCursor: GraphQLTweetFoundResponse; + + try { + loadCursor = await fetchTwitterThread(id, request.event, true, cursor.value) + + if (typeof loadCursor?.data?.threaded_conversation_with_injections_v2?.instructions === 'undefined') { + console.log('Unknown data while fetching cursor', loadCursor); + break; + } + } catch(e) { + console.log('Error fetching cursor', e); + break; + } + const cursorResponse = processResponse(loadCursor.data.threaded_conversation_with_injections_v2.instructions); + bucket.tweets = cursorResponse.tweets.concat(filterBucketTweets(bucket.tweets, originalTweet)); + /* Remove old cursor and add new top cursor if necessary */ + consolidateCursors(bucket.cursors, cursorResponse.cursors); + + // console.log('updated bucket of tweets', bucket.tweets); + console.log('updated bucket of cursors', bucket.cursors); + } + + console.log('Preview of previous tweet:', findPreviousTweet(currentId, bucket)); + } + + const socialThread: SocialThread = { + post: post, + thread: [], + author: author + } + + threadTweets.forEach(async (tweet) => { + socialThread.thread?.push(await buildAPITweet(tweet, undefined, true, false)); + }); + + return socialThread; +} + +export const threadAPIProvider = async (request: IRequest) => { + const { id } = request.params; + + const processedResponse = await processTwitterThread(id, true, request); + + return new Response(JSON.stringify(processedResponse), { + headers: { ...Constants.RESPONSE_HEADERS, ...Constants.API_RESPONSE_HEADERS } + }) +} \ No newline at end of file diff --git a/src/types/twitterTypes.d.ts b/src/types/twitterTypes.d.ts index 35bce9b..8757117 100644 --- a/src/types/twitterTypes.d.ts +++ b/src/types/twitterTypes.d.ts @@ -354,7 +354,7 @@ type GraphQLTweetLegacy = { type GraphQLTweet = { // Workaround result: GraphQLTweet; - __typename: 'Tweet' | 'TweetUnavailable'; + __typename: 'Tweet' | 'TweetWithVisibilityResults' | 'TweetUnavailable'; rest_id: string; // "1674824189176590336", has_birdwatch_notes: false; core: { @@ -444,37 +444,76 @@ type TweetTombstone = { }; }; }; + +type GraphQLTimelineTweet = { + item: 'TimelineTweet'; + __typename: 'TimelineTweet'; + tweet_results: { + result: GraphQLTweet | TweetTombstone; + }; +} + +type GraphQLTimelineCursor = { + cursorType: 'Top' | 'Bottom' | 'ShowMoreThreadsPrompt' | 'ShowMore'; + itemType: 'TimelineTimelineCursor'; + value: string; + __typename: 'TimelineTimelineCursor'; +} + +interface GraphQLBaseTimeline { + entryType: string; + __typename: string; +} + +type GraphQLTimelineItem = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineItem'; + __typename: 'TimelineTimelineItem'; + itemContent: GraphQLTimelineTweet | GraphQLTimelineCursor; +} + +type GraphQLTimelineModule = GraphQLBaseTimeline & { + entryType: 'TimelineTimelineModule'; + __typename: 'TimelineTimelineModule'; + items: { + entryId: `conversationthread-${number}-tweet-${number}`; + item: GraphQLTimelineItem + }[]; +} + type GraphQLTimelineTweetEntry = { /** The entryID contains the tweet ID */ entryId: `tweet-${number}`; // "tweet-1674824189176590336" sortIndex: string; - content: { - entryType: 'TimelineTimelineItem'; - __typename: 'TimelineTimelineItem'; - itemContent: { - item: 'TimelineTweet'; - __typename: 'TimelineTweet'; - tweet_results: { - result: GraphQLTweet | TweetTombstone; - }; - }; - }; + content: GraphQLTimelineItem; }; + +type GraphQLModuleTweetEntry = { + /** The entryID contains the tweet ID */ + sortIndex: string; + item: GraphQLTimelineItem | GraphQLTimelineModule; +}; + type GraphQLConversationThread = { entryId: `conversationthread-${number}`; // "conversationthread-1674824189176590336" sortIndex: string; + content: GraphQLTimelineModule; }; type GraphQLTimelineEntry = GraphQLTimelineTweetEntry | GraphQLConversationThread | unknown; -type V2ThreadInstruction = TimeLineAddEntriesInstruction | TimeLineTerminateTimelineInstruction; +type V2ThreadInstruction = TimelineAddEntriesInstruction | TimelineTerminateTimelineInstruction | TimelineAddModulesInstruction; -type TimeLineAddEntriesInstruction = { +type TimelineAddEntriesInstruction = { type: 'TimelineAddEntries'; entries: GraphQLTimelineEntry[]; }; -type TimeLineTerminateTimelineInstruction = { +type TimelineAddModulesInstruction = { + type: 'TimelineAddToModule'; + moduleItems: GraphQLTimelineEntry[]; +}; + +type TimelineTerminateTimelineInstruction = { type: 'TimelineTerminateTimeline'; direction: 'Top'; }; diff --git a/src/types/types.d.ts b/src/types/types.d.ts index d6f9f7f..e911d81 100644 --- a/src/types/types.d.ts +++ b/src/types/types.d.ts @@ -81,14 +81,6 @@ interface APITranslate { target_lang: string; } -interface BaseUser { - id?: string; - name?: string; - screen_name?: string; - avatar_url?: string; - banner_url?: string; -} - interface APIExternalMedia { type: 'video'; url: string; @@ -136,7 +128,7 @@ interface APIMosaicPhoto extends APIMedia { }; } -interface APITweet { +interface APIPost { id: string; url: string; text: string; @@ -144,15 +136,11 @@ interface APITweet { created_timestamp: number; likes: number; - retweets: number; + reposts: number; replies: number; - views?: number | null; - color: string | null; - - quote?: APITweet; + quote?: APIPost; poll?: APIPoll; - translation?: APITranslate; author: APIUser; media?: { @@ -169,24 +157,41 @@ interface APITweet { replying_to: string | null; replying_to_status: string | null; - source: string; + reply_of: { + screen_name: string | null; + post: string | null; + } | null - is_note_tweet: boolean; + source: string | null; - twitter_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; + embed_card: 'tweet' | 'summary' | 'summary_large_image' | 'player'; } -interface APIUser extends BaseUser { +interface APITweet extends APIPost { + retweets: number; + views?: number | null; + translation?: APITranslate; + + is_note_tweet: boolean; +} + +interface APIUser { + id: string; + name: string; + screen_name: string; + global_screen_name: string; + avatar_url: string | null; + banner_url: string | null; // verified: 'legacy' | 'blue'| 'business' | 'government'; // verified_label: string; - description: string; - location: string; + description: string | null; + location: string | null; url: string; - avatar_color?: string | null; protected: boolean; followers: number; following: number; - tweets: number; + tweets?: number; + posts?: number; likes: number; joined: string; website: { diff --git a/src/worker.ts b/src/worker.ts index f3435f9..83f3d18 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -10,6 +10,7 @@ import { Strings } from './strings'; import motd from '../motd.json'; import { sanitizeText } from './helpers/utils'; import { handleProfile } from './user'; +import { threadAPIProvider } from './providers/twitter/status'; declare const globalThis: { fetchCompletedTime: number; @@ -453,6 +454,7 @@ router.get('/status/:id', statusRequest); router.get('/status/:id/:language', statusRequest); router.get('/version', versionRequest); router.get('/set_base_redirect', setRedirectRequest); +router.get('/v2/twitter/thread/:id', threadAPIProvider) /* Oembeds (used by Discord to enhance responses) From 8ce632308e502291a560a09d7de17cb551ee6b6a Mon Sep 17 00:00:00 2001 From: dangered wolf Date: Thu, 19 Oct 2023 01:24:55 -0400 Subject: [PATCH 4/4] upstream fixes --- src/api/status.ts | 7 ++++++- src/providers/twitter/processor.ts | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/api/status.ts b/src/api/status.ts index ef129c7..9a8d4f5 100644 --- a/src/api/status.ts +++ b/src/api/status.ts @@ -16,7 +16,7 @@ const populateTweetProperties = async ( conversation: TweetResultsByRestIdResult, // TimelineBlobPartial, language: string | undefined // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { const apiTweet = {} as APITweet; /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. @@ -38,6 +38,11 @@ const populateTweetProperties = async ( tweet.views = tweet?.tweet?.views; } + if (typeof tweet.core === 'undefined') { + console.log('Tweet still not valid', tweet); + return null; + } + /* With v2 conversation API we re-add the user object ot the tweet because Twitter stores it separately in the conversation API. This is to consolidate it in case a user appears multiple times in a thread. */ diff --git a/src/providers/twitter/processor.ts b/src/providers/twitter/processor.ts index d28203b..3b8e1b7 100644 --- a/src/providers/twitter/processor.ts +++ b/src/providers/twitter/processor.ts @@ -13,7 +13,7 @@ export const buildAPITweet = async ( threadPiece = false, legacyAPI = false // eslint-disable-next-line sonarjs/cognitive-complexity -): Promise => { +): Promise => { const apiTweet = {} as APITweet; /* Sometimes, Twitter returns a different kind of Tweet type called 'TweetWithVisibilityResults'. @@ -35,6 +35,11 @@ export const buildAPITweet = async ( tweet.views = tweet?.tweet?.views; } + if (typeof tweet.core === 'undefined') { + console.log('Tweet still not valid', tweet); + return null; + } + const graphQLUser = tweet.core.user_results.result; const apiUser = convertToApiUser(graphQLUser); @@ -124,6 +129,16 @@ export const buildAPITweet = async ( photos: [], videos: [], }; + + /* We found a quote tweet, let's process that too */ + const quoteTweet = tweet.quoted_status_result; + if (quoteTweet) { + apiTweet.quote = (await buildAPITweet(quoteTweet, language)) as APITweet; + /* Only override the embed_card if it's a basic tweet, since media always takes precedence */ + if (apiTweet.embed_card === 'tweet'&& apiTweet.quote !== null) { + apiTweet.embed_card = apiTweet.quote.embed_card; + } + } const mediaList = Array.from( tweet.legacy.extended_entities?.media || tweet.legacy.entities?.media || []