User-agent: *
crawl-delay: 30

User-agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; http://megaindex.com/crawler)
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: WebMeUp Crawler
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: FAST Crawler
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baidu Spider
Disallow: /

User-agent: PDFDriveCrawler 2.36 ( http://www.pdfdrive.net )
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: ChangeDetection Robot
Disallow: /

User-agent: Mozilla/5.0 (compatible) SemanticScholarBot ( https://www.semanticscholar.org/crawler)
Disallow: /

User-Agent: POLITICO-EU
Disallow: /