DevComp/fxtwitter-docker
1
0
Fork 0
mirror of https://github.com/CompeyDev/fxtwitter-docker.git synced 2025-04-05 18:40:56 +01:00
Code Issues Projects Releases Packages Wiki Activity

Block Yandex in robots.txt

Browse source
This commit is contained in:
dangered wolf 2023-04-18 21:26:44 -04:00
parent 0a81ef90d6
commit c323e2f8d5
No known key found for this signature in database
GPG key ID: 41E4D37680ED8B58
1 changed files with 5 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/constants.ts
View file

@ -63,7 +63,11 @@ export const Constants = {
}, },
POLL_TWEET_CACHE: 'max-age=60', POLL_TWEET_CACHE: 'max-age=60',
DEFAULT_COLOR: '#10A3FF', DEFAULT_COLOR: '#10A3FF',
ROBOTS_TXT: `User-agent: * ROBOTS_TXT: `# Yandex crawls far, far heavier than Googlebot and Bingbot combined
User-agent: YandexBot
Disallow: /
User-agent: *
Allow: /$ Allow: /$
Allow: /*/status Allow: /*/status
Allow: /*/status/ Allow: /*/status/