# Tier 0
User-agent: Googlebot
User-agent: Googlebot-Mobile
User-agent: Yandex # Russia
Disallow: /admin/
Allow: /

# Tier 1
User-agent: Bingbot # Bing
User-agent: Slurp # Yahoo
User-agent: coccocbot # Vietnam
User-agent: DuckDuckBot # DuckDuckGo
User-agent: baiduspider # China
User-agent: Applebot # Apple
Disallow: /admin/
Allow: /

# Tier 1.5
User-agent: SeznamBot # Czech Republic
User-agent: NaverBot # South Korea
User-agent: Yeti # South Korea
User-agent: Mail.ru # Russia
User-agent: Sogou # China
User-agent: 360Spider # China
User-agent: Neevabot # Neeva
Disallow: /admin/
Allow: /
Crawl-delay: 1

# Tier 2
User-agent: Teoma # Ask
User-agent: SputnikBot # Russia
User-agent: Daumoa # South Korea
User-agent: Dazoobot # France
User-agent: DeuSu # Germany
User-agent: EuripBot # Europe
User-agent: Exploratodo # Latin America
User-agent: istellabot # Italy
User-agent: moget # Japan
User-agent: ichiro # Japan
Disallow: /admin/
Allow: /
Crawl-delay: 10

# Tier 2.5
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Googlebot-Image
User-agent: GoogleOther
User-agent: AdsBot-Google
User-agent: BingPreview
User-agent: msnbot
User-agent: msnbot-media
Disallow: /admin/
Allow: /
#Crawl-delay: 90

# Ads
User-agent: Mediapartners-Google
User-agent: ias_crawler
User-agent: GrapeshotCrawler
User-agent: CriteoBot
User-agent: TTD-Content
User-agent: AmazonAdBot
User-agent: Leikibot
User-agent: DoubleVerify
User-agent: Opebot-v (https://www.1plusx.com)
Disallow: /admin/
Allow: /

# Buggy
User-agent: Petalbot # Huawei
User-agent: weborama-fetcher
Disallow: /admin/
Disallow: *?*
Disallow: /
#Allow: /
#Crawl-delay: 60

# Blocked
User-agent: Amazonbot
User-agent: Bytespider
User-agent: CCBot
User-agent: ClaudeBot
User-agent: meta-externalagent
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: BLEXBot
User-agent: EzoicBot
User-agent: WellKnownBot
User-agent: GPTBot
User-agent: Cincraw
User-Agent: PerplexityBot
User-Agent: Google-Extended
User-Agent: Applebot-Extended
User-Agent: PanguBot
User-Agent: YouBot
User-Agent: YandexAdditionalBot
User-Agent: similarweb
User-Agent: MJ12Bot
Disallow: /

User-agent: *
Disallow: /

Sitemap: https://www.chemicalaid.com/sitemap.php