# robots.txt for ashwingopalsamy.in
# Robots Exclusion Protocol, RFC 9309 (https://www.rfc-editor.org/rfc/rfc9309).
#
# Open by default: any crawler not named below may fetch the whole site.
# Content-usage and attribution terms for AI systems are declared separately
# in /ai.txt (CC BY 4.0, attribution required), and the machine-readable
# briefing lives at /ai, /llms.txt, and /knowledge.json.

# ---- Search, AI, and social crawlers: explicitly welcome ----
# One group, many agents (RFC 9309 allows repeated User-agent lines to share a
# single rule set). Named so their access is intentional and auditable rather
# than incidental.
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Amazonbot
User-agent: KimiBot
User-agent: Kimi-User
User-agent: ZhipuBot
User-agent: Zhipu-User
User-agent: MistralAI-User
User-agent: AverbisBot
User-agent: PreonBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: YouBot
User-agent: DuckAssistBot
User-agent: Timpibot
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

# ---- Weak-attribution bulk scrapers: blocked ----
# High-volume crawlers that feed downstream corpora with no reliable
# attribution back to the source. Blocked deliberately.
User-agent: Bytespider
User-agent: CCBot
Content-Signal: ai-train=no, search=no, ai-input=no
Disallow: /

# ---- Everyone else: allowed ----
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /

Sitemap: https://ashwingopalsamy.in/sitemap.xml
# Agentmap: https://ashwingopalsamy.in/.well-known/ai-catalog.json