# robots.txt for https://ai-risks.co.uk/
# Purpose: maximise legitimate search and AI discovery while discouraging
# selected low-value bulk crawlers.
#
# Important: robots.txt is an advisory crawling policy, not a security control.
# Use CDN/WAF rules for abusive or spoofed bots.

# ------------------------------------------------------------
# Major search and AI discovery / retrieval crawlers
# ------------------------------------------------------------

User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: OAI-AdsBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: CCBot
Allow: /

# ------------------------------------------------------------
# Selected low-value / bulk crawling bots
# These are not required for Google, Bing, ChatGPT, Claude,
# Perplexity or Apple search visibility.
# ------------------------------------------------------------

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

# ------------------------------------------------------------
# All other compliant crawlers
# Keep the public site crawlable by default.
# ------------------------------------------------------------

User-agent: *
Allow: /

# XML sitemap
Sitemap: https://ai-risks.co.uk/sitemap.xml