#robots.txt file for www.the-scientist.com
User-agent: *
Disallow: /api/
Disallow: /account
Disallow: /digest/view/
Disallow: /preview
Disallow: /login
Disallow: /search

Disallow: /ts-digest/issue/.*-(?:1[7-9]|[2-9]d+)-d+$

#bot crawl-delays
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: Bingbot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: YandexBot
Crawl-delay: 5

User-agent: Yandex
Crawl-delay: 5

#sitemap list for www.the-scientist.com
Sitemap: https://www.the-scientist.com/sitemap/sitemap.xml
Sitemap: https://www.the-scientist.com/sitemap/sitemap-news.xml