User-agent: *
Allow: /

# Admin is auth-gated (401) regardless; this keeps it out of the index.
Disallow: /admin/

# Crawl budget. Googlebot spent 21% of its visits here over 30 days on JSON that
# cannot rank — /api/map-data, /api/featured and the chat asset bundles. The
# discovery surfaces an agent actually needs (/api/info, /openapi.json,
# /.well-known/) stay open, and named AI crawlers below are unaffected because a
# more specific group always wins.
Disallow: /api/map-data
Disallow: /api/featured
Disallow: /assets/
# The metered analysis endpoints. Measured 2 Sep 2026: Googlebot spent 19 of its
# crawls on 402 paywall responses - 11 on map-data (already blocked above) plus
# cex-funding and trade-analysis, which were not. A crawler cannot rank a 402 and
# should not be spending budget discovering that. Agents reach these through
# openapi.json and the MCP tool list, neither of which is affected.
Disallow: /api/cex-funding
Disallow: /api/trade-analysis
Disallow: /api/cohorts
Disallow: /api/scan-cabal

# AI crawlers — explicitly allowed for discovery & retrieval
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: CCBot
Allow: /
User-agent: Applebot-Extended
Allow: /


Sitemap: https://cabal-hunter.com/sitemap.xml
Sitemap: https://cabal-hunter.com/sitemap-deployers.xml