User-agent: *
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: meta-webindexer
Disallow: /portal/
Disallow: /admin/
Disallow: /submit/
Disallow: /report/
Disallow: /share/
Disallow: /api/internal/
Disallow: /api/e
Allow: /api/v1/
Allow: /api/oembed
Allow: /api/docs
Allow: /mcp
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/

# AI crawlers: the search and assistant crawlers named above are welcome on purpose and follow the
# same rules as everyone else. Training crawlers (GPTBot, ClaudeBot, CCBot, Google-Extended,
# Applebot-Extended and others) are not blocked either: they follow the User-agent: * rules.

# Machine-readable interfaces (comments, not directives)
# llms.txt: https://arcadeindex.io/llms.txt
# llms-full.txt: https://arcadeindex.io/llms-full.txt
# MCP (Streamable HTTP): https://arcadeindex.io/mcp
# OpenAPI: https://arcadeindex.io/api/v1/openapi.json
# Agent manifest: https://arcadeindex.io/.well-known/arcadeindex.json

Sitemap: https://arcadeindex.io/sitemaps/sitemap-index.xml