# apexclawai.com — AI crawler posture, reviewed 2026-08-07
# Policy: OBSERVE, NOT BLOCK. Blocking unverified agents before verification is
# widely adopted costs legitimate AI citations for no real security gain today.
# Crawlers are grouped by the four jobs they actually do: agent (live user fetch),
# retrieval (answer-index build), search (classic index feeding AI), training.
# Operator surfaces are excluded from every group.

# ---------- RETRIEVAL / ANSWER-INDEX (highest citation value) ----------
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: DuckAssistBot
Allow: /
Disallow: /mc/
Disallow: /admin/
Disallow: /omega

# ---------- AGENT / LIVE USER FETCH ----------
User-agent: ChatGPT-User
User-agent: Perplexity-User
User-agent: Claude-User
Allow: /
Disallow: /mc/
Disallow: /admin/
Disallow: /omega

# ---------- TRAINING ----------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: CCBot
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Diffbot
User-agent: Timpibot
User-agent: Omgilibot
User-agent: YouBot
Allow: /
Disallow: /mc/
Disallow: /admin/
Disallow: /omega

# ---------- CLASSIC SEARCH ----------
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: Slurp
Allow: /
Disallow: /mc/
Disallow: /admin/
Disallow: /omega

# ---------- DEFAULT ----------
User-agent: *
Allow: /
Disallow: /mc/
Disallow: /admin/
Disallow: /omega

# Machine-readable surfaces, built for agents rather than scraped by them:
#   /llms.txt                            curated index
#   /llms-full.txt                       full-text corpus, single file
#   /.well-known/agent.json              capability manifest
#   /.well-known/apexclaw-receipts.schema.json         ApexClaw Receipts v1 (governance-audit receipt format, CC-BY, DRAFT)
#   /.well-known/answers.json            direct answers index
#   /.well-known/claims-ledger.json      material product/evidence claims with verification + expiry dates
#   /.well-known/security.txt            RFC 9116
# Agent identity verification: /.well-known/http-message-signatures-directory (RFC 9421)

Sitemap: https://apexclawai.com/sitemap.xml