{
  "version": "1.0.3",
  "updated": "2025-12-29",
  "crawlers": [
    {
      "name": "GPTBot",
      "company": "OpenAI",
      "user_agent": "GPTBot",
      "purpose": "ChatGPT training & web browsing",
      "recommended": true,
      "docs_url": "https://platform.openai.com/docs/gptbot"
    },
    {
      "name": "ChatGPT-User",
      "company": "OpenAI",
      "user_agent": "ChatGPT-User",
      "purpose": "ChatGPT browse mode (real-time)",
      "recommended": true,
      "docs_url": "https://platform.openai.com/docs/gptbot"
    },
    {
      "name": "OAI-SearchBot",
      "company": "OpenAI",
      "user_agent": "OAI-SearchBot",
      "purpose": "ChatGPT Search feature",
      "recommended": true,
      "docs_url": "https://platform.openai.com/docs/bots"
    },
    {
      "name": "Google-Extended",
      "company": "Google",
      "user_agent": "Google-Extended",
      "purpose": "Gemini/Bard training",
      "recommended": true,
      "docs_url": "https://developers.google.com/search/docs/crawling-indexing/google-extended"
    },
    {
      "name": "Gemini-Deep-Research",
      "company": "Google",
      "user_agent": "Gemini-Deep-Research",
      "purpose": "Gemini deep research feature",
      "recommended": true,
      "docs_url": "https://blog.google/products/gemini/google-gemini-deep-research/"
    },
    {
      "name": "Google-NotebookLM",
      "company": "Google",
      "user_agent": "Google-NotebookLM",
      "purpose": "NotebookLM source fetching",
      "recommended": true,
      "docs_url": "https://notebooklm.google/"
    },
    {
      "name": "GoogleOther",
      "company": "Google",
      "user_agent": "GoogleOther",
      "purpose": "Google AI research and development",
      "recommended": false,
      "docs_url": "https://developers.google.com/search/docs/crawling-indexing/googleother"
    },
    {
      "name": "Google-CloudVertexBot",
      "company": "Google",
      "user_agent": "Google-CloudVertexBot",
      "purpose": "Vertex AI platform",
      "recommended": false,
      "docs_url": "https://cloud.google.com/vertex-ai"
    },
    {
      "name": "anthropic-ai",
      "company": "Anthropic",
      "user_agent": "anthropic-ai",
      "purpose": "Claude training",
      "recommended": true,
      "docs_url": "https://www.anthropic.com/robots-txt"
    },
    {
      "name": "ClaudeBot",
      "company": "Anthropic",
      "user_agent": "ClaudeBot",
      "purpose": "Claude web access & citations",
      "recommended": true,
      "docs_url": "https://www.anthropic.com/robots-txt"
    },
    {
      "name": "Claude-Web",
      "company": "Anthropic",
      "user_agent": "Claude-Web",
      "purpose": "Claude web features",
      "recommended": true,
      "docs_url": "https://www.anthropic.com/robots-txt"
    },
    {
      "name": "Claude-User",
      "company": "Anthropic",
      "user_agent": "Claude-User",
      "purpose": "User-triggered page fetches for Claude",
      "recommended": true,
      "docs_url": "https://www.anthropic.com/robots-txt"
    },
    {
      "name": "Claude-SearchBot",
      "company": "Anthropic",
      "user_agent": "Claude-SearchBot",
      "purpose": "Claude search result quality",
      "recommended": true,
      "docs_url": "https://www.anthropic.com/robots-txt"
    },
    {
      "name": "PerplexityBot",
      "company": "Perplexity",
      "user_agent": "PerplexityBot",
      "purpose": "AI search engine indexing",
      "recommended": true,
      "docs_url": "https://docs.perplexity.ai/docs/perplexitybot"
    },
    {
      "name": "Perplexity-User",
      "company": "Perplexity",
      "user_agent": "Perplexity-User",
      "purpose": "User-triggered real-time fetches",
      "recommended": true,
      "docs_url": "https://docs.perplexity.ai/docs/perplexitybot"
    },
    {
      "name": "xAI-Grok-Bot",
      "company": "xAI",
      "user_agent": "xAI-Grok-Bot",
      "purpose": "Grok training & citations",
      "recommended": true,
      "docs_url": "https://x.ai"
    },
    {
      "name": "Applebot",
      "company": "Apple",
      "user_agent": "Applebot",
      "purpose": "Siri and Spotlight search",
      "recommended": true,
      "docs_url": "https://support.apple.com/en-us/HT204683"
    },
    {
      "name": "Applebot-Extended",
      "company": "Apple",
      "user_agent": "Applebot-Extended",
      "purpose": "Apple Intelligence features",
      "recommended": true,
      "docs_url": "https://support.apple.com/en-us/HT204683"
    },
    {
      "name": "Meta-ExternalAgent",
      "company": "Meta",
      "user_agent": "Meta-ExternalAgent",
      "purpose": "Meta AI training",
      "recommended": false,
      "docs_url": "https://www.facebook.com/robots.txt"
    },
    {
      "name": "FacebookBot",
      "company": "Meta",
      "user_agent": "FacebookBot",
      "purpose": "Meta AI features",
      "recommended": false,
      "docs_url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"
    },
    {
      "name": "Amazonbot",
      "company": "Amazon",
      "user_agent": "Amazonbot",
      "purpose": "Alexa and Amazon AI",
      "recommended": true,
      "docs_url": "https://developer.amazon.com/amazonbot"
    },
    {
      "name": "amazon-kendra",
      "company": "Amazon",
      "user_agent": "amazon-kendra",
      "purpose": "Amazon Kendra enterprise search",
      "recommended": false,
      "docs_url": "https://aws.amazon.com/kendra/"
    },
    {
      "name": "Bravebot",
      "company": "Brave",
      "user_agent": "Bravebot",
      "purpose": "Brave Search and Leo AI",
      "recommended": true,
      "docs_url": "https://search.brave.com/help/bravebot"
    },
    {
      "name": "CCBot",
      "company": "Common Crawl",
      "user_agent": "CCBot",
      "purpose": "Open dataset used by many LLMs",
      "recommended": true,
      "docs_url": "https://commoncrawl.org/ccbot"
    },
    {
      "name": "DuckAssistBot",
      "company": "DuckDuckGo",
      "user_agent": "DuckAssistBot",
      "purpose": "DuckDuckGo AI Assist",
      "recommended": true,
      "docs_url": "https://duckduckgo.com/duckduckbot"
    },
    {
      "name": "Bytespider",
      "company": "ByteDance",
      "user_agent": "Bytespider",
      "purpose": "TikTok/Doubao AI training",
      "recommended": false,
      "docs_url": null
    },
    {
      "name": "cohere-ai",
      "company": "Cohere",
      "user_agent": "cohere-ai",
      "purpose": "Enterprise AI training",
      "recommended": false,
      "docs_url": "https://cohere.com"
    },
    {
      "name": "Deepseek",
      "company": "DeepSeek",
      "user_agent": "Deepseek",
      "purpose": "DeepSeek AI training",
      "recommended": false,
      "docs_url": "https://deepseek.com"
    },
    {
      "name": "DeepseekBot",
      "company": "DeepSeek",
      "user_agent": "DeepseekBot",
      "purpose": "DeepSeek web crawling",
      "recommended": false,
      "docs_url": "https://deepseek.com"
    },
    {
      "name": "MistralAI-User",
      "company": "Mistral",
      "user_agent": "MistralAI-User",
      "purpose": "Mistral AI (European)",
      "recommended": false,
      "docs_url": "https://mistral.ai"
    },
    {
      "name": "Groq-Bot",
      "company": "Groq",
      "user_agent": "Groq-Bot",
      "purpose": "Groq AI inference platform",
      "recommended": false,
      "docs_url": "https://groq.com"
    },
    {
      "name": "Diffbot",
      "company": "Diffbot",
      "user_agent": "Diffbot",
      "purpose": "Knowledge graph construction",
      "recommended": false,
      "docs_url": "https://www.diffbot.com/company/crawling-policy"
    },
    {
      "name": "YouBot",
      "company": "You.com",
      "user_agent": "YouBot",
      "purpose": "AI search engine",
      "recommended": true,
      "docs_url": "https://about.you.com/youbot"
    },
    {
      "name": "PhindBot",
      "company": "Phind",
      "user_agent": "PhindBot",
      "purpose": "Phind AI search for developers",
      "recommended": true,
      "docs_url": "https://www.phind.com"
    },
    {
      "name": "ExaBot",
      "company": "Exa",
      "user_agent": "ExaBot",
      "purpose": "Exa semantic search indexing",
      "recommended": true,
      "docs_url": "https://exa.ai"
    },
    {
      "name": "AndiBot",
      "company": "Andi",
      "user_agent": "AndiBot",
      "purpose": "Andi conversational search",
      "recommended": true,
      "docs_url": "https://andisearch.com"
    },
    {
      "name": "LinkedInBot",
      "company": "LinkedIn",
      "user_agent": "LinkedInBot",
      "purpose": "LinkedIn link previews",
      "recommended": true,
      "docs_url": "https://www.linkedin.com/help/linkedin"
    },
    {
      "name": "ImagesiftBot",
      "company": "Imagesift",
      "user_agent": "ImagesiftBot",
      "purpose": "Image AI training",
      "recommended": false,
      "docs_url": null
    }
  ]
}