# Soclo — https://soclo.app
#
# Everything here is open to everyone, including AI crawlers. The named blocks
# below say nothing different from the wildcard; they exist because a crawler
# that finds its own user-agent stops reading the wildcard, and because being
# explicit is clearer than being silent about whether AI systems are welcome.
#
# Two kinds of AI agent are named. TRAINING crawlers (GPTBot, ClaudeBot,
# Google-Extended, Applebot-Extended, CCBot, Amazonbot, Meta-ExternalAgent,
# cohere-ai, Bytespider) collect text that may inform future models. RETRIEVAL
# agents (OAI-SearchBot, ChatGPT-User, Claude-User, PerplexityBot,
# Perplexity-User) fetch pages to answer a question being asked right now —
# those are the ones that decide whether Soclo can be cited in an answer.

User-agent: *
Allow: /

# --- retrieval: fetches a page to answer a live question -------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# --- training / corpus -----------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

Sitemap: https://soclo.app/sitemap-index.xml
Sitemap: https://soclo.app/video-sitemap.xml