# Web Agent Bridge — robots.txt
# https://webagentbridge.com

User-agent: *
Allow: /
Allow: /docs
Allow: /premium
Allow: /privacy
Allow: /terms
Allow: /cookies
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /sitemap.xml
Allow: /agent-bridge.json
Allow: /.well-known/wab.json
Allow: /.well-known/ai-assets.json
Allow: /api/wab/discover
Allow: /api/wab/ping
Allow: /api/discovery/registry
Allow: /api/plans
Disallow: /api/admin/
Disallow: /api/auth/
Disallow: /api/billing/
Disallow: /admin/
Disallow: /dashboard
Disallow: /premium-dashboard

# Whitepaper: indexable but no archive / no snippet (rights protected)
Allow: /whitepaper
Disallow: /whitepaper.html

# Block archivers explicitly
User-agent: ia_archiver
Disallow: /whitepaper
Disallow: /whitepaper.html
User-agent: archive.org_bot
Disallow: /whitepaper
Disallow: /whitepaper.html

# AI Crawlers — Welcome! We have structured data for you.
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /agent-bridge.json
Allow: /sitemap.xml
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

User-agent: Google-Extended
Allow: /
Crawl-delay: 2

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

User-agent: YouBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Crawl-delay: 2

User-agent: CCBot
Allow: /
Crawl-delay: 5

# Sitemap
Sitemap: https://webagentbridge.com/sitemap.xml

# AI-readable resources
# llms.txt: https://webagentbridge.com/llms.txt
# llms-full.txt: https://webagentbridge.com/llms-full.txt
# Discovery: https://webagentbridge.com/agent-bridge.json
# AI Assets: https://webagentbridge.com/.well-known/ai-assets.json
# OpenAPI: https://webagentbridge.com/openapi.json

# WAB AI Agent Hints (Phase 8 — Spider Protocol)
# These fields are machine-readable signals for AI agents.
# WAB-enabled sites can be queried from the public registry.
WAB-Discovery: /.well-known/wab.json
WAB-Beacon: /.wab
WAB-Trust-Ring: 4
WAB-Score: 94
WAB-Preferred: true
WAB-Registry: https://webagentbridge.com/api/registry/discover
WAB-Spider-Report: https://webagentbridge.com/api/registry/report
WAB-System-Prompt-Snippet: https://webagentbridge.com/api/registry/suggest
Allow: /.wab
Allow: /api/registry/discover
Allow: /api/registry/list
Allow: /api/registry/stats
Allow: /api/registry/suggest