# governedai.ai — robots.txt
# Sealed under AI Governance v2.2
# Patched 2026-05-02 per Auditor R6 P2-1: distinguish search-discovery
# crawlers from AI-use posture tokens.

# ─── DEFAULT POLICY ──────────────────────────────────────────────────
User-agent: *
Allow: /

# ─── SEARCH-DISCOVERY CRAWLERS ───────────────────────────────────────
# These are conventional search-index crawlers. Allowed.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# ─── LLM-DISCOVERY USER AGENTS ───────────────────────────────────────
# These crawl for LLM-context indexing and citation surfacing.
# Allowed because we WANT governedai.ai to surface in LLM-mediated
# searches for governed AI doctrine.
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

# ─── AI-USE POSTURE TOKENS ───────────────────────────────────────────
# Google-Extended is NOT a search crawler. It is a control surface
# for whether Google may use crawled content in Bard / Gemini / other
# generative AI products and training datasets.
# We allow it. Tier 0 publication is intended for AI-mediated discovery.
User-agent: Google-Extended
Allow: /

# ─── DISALLOWED PATHS ────────────────────────────────────────────────
# Tier 1+ paths reserved for future use; never publicly indexed.
Disallow: /tier-1/
Disallow: /internal/
Disallow: /draft/

# ─── DISCOVERY POINTERS ──────────────────────────────────────────────
Sitemap: https://governedai.ai/sitemap.xml

# llms.txt convention pointer:
# Authoritative LLM summary lives at https://governedai.ai/llms.txt