# =============================================================
# robots.txt — Strategy Collective (strategyco.com)
# =============================================================
# Sitemap declared first per crawler convention.
Sitemap: https://strategyco.com/sitemap.xml

# =============================================================
# SEARCH ENGINE BOTS — Full access to public content
# =============================================================
User-agent: Googlebot
Allow: /
Disallow: /api/
Disallow: /downloads/_archive/
Disallow: /memories/
Disallow: /_pdf_fonts/
Allow: /downloads/*.pdf

User-agent: Bingbot
Allow: /
Disallow: /api/
Disallow: /downloads/_archive/
Disallow: /memories/
Disallow: /_pdf_fonts/
Allow: /downloads/*.pdf

User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /memories/

# =============================================================
# AI TRAINING + ANSWER ENGINE BOTS — Welcome (GEO strategy)
# These bots crawl content to train models or generate AI answers.
# Allowing them maximizes reach in emerging AI-search surfaces.
# =============================================================

# OpenAI
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /memories/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /memories/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /memories/

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /memories/

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /memories/

User-agent: Anthropic-AI
Allow: /
Disallow: /api/
Disallow: /memories/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /memories/

# Apple AI (Siri, Apple Intelligence)
User-agent: Applebot-Extended
Allow: /
Disallow: /api/
Disallow: /memories/

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /api/
Disallow: /memories/

# Common Crawl (used by many open-source LLM training datasets)
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /memories/

# =============================================================
# SEO TOOL BOTS — Blocked
# Ahrefs / Semrush / Majestic / DotBot consume crawl budget without
# adding indexing or AI visibility. Our own audit data on strategyco.com
# via those platforms comes from their panel data, not from crawling
# our site, so this has no cost to internal reporting.
# =============================================================
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# =============================================================
# ARCHIVE BOTS — Allow (historical credibility signal for AI)
# =============================================================
User-agent: ia_archiver
Allow: /
Disallow: /memories/

# =============================================================
# DEFAULT — All other bots
# =============================================================
User-agent: *
Allow: /
Disallow: /api/
Disallow: /downloads/_archive/
Disallow: /memories/
Disallow: /_pdf_fonts/
Allow: /downloads/*.pdf