{
  "registryVersion": "2026-08-13",
  "agents": [
    {
      "slug": "gptbot",
      "token": "GPTBot",
      "operator": "OpenAI",
      "purpose": "training",
      "isFetcher": true,
      "description": "OpenAI's automatic crawler for content that may be used to improve and train generative AI foundation models.",
      "sourceUrl": "https://developers.openai.com/api/docs/bots",
      "sourceCheckedAt": "2026-08-13"
    },
    {
      "slug": "oai-searchbot",
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "purpose": "search",
      "isFetcher": true,
      "description": "OpenAI's automatic search crawler, used to surface and link websites in ChatGPT search results.",
      "sourceUrl": "https://developers.openai.com/api/docs/bots",
      "sourceCheckedAt": "2026-08-13"
    },
    {
      "slug": "chatgpt-user",
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "purpose": "user_action",
      "isFetcher": true,
      "description": "OpenAI's user-initiated fetcher for certain ChatGPT, Custom GPT, and GPT Action requests.",
      "sourceUrl": "https://developers.openai.com/api/docs/bots",
      "sourceCheckedAt": "2026-08-13",
      "notes": "OpenAI says this is not an automatic web crawler, robots.txt rules may not apply, and it does not control ChatGPT Search inclusion."
    },
    {
      "slug": "claudebot",
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "purpose": "training",
      "isFetcher": true,
      "description": "Anthropic's automatic crawler for web content that could contribute to model training.",
      "sourceUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "sourceCheckedAt": "2026-08-13"
    },
    {
      "slug": "claude-searchbot",
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "purpose": "search",
      "isFetcher": true,
      "description": "Anthropic's automatic crawler for improving the relevance and accuracy of Claude search results.",
      "sourceUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "sourceCheckedAt": "2026-08-13"
    },
    {
      "slug": "claude-user",
      "token": "Claude-User",
      "operator": "Anthropic",
      "purpose": "user_action",
      "isFetcher": true,
      "description": "Anthropic's user-directed fetcher for retrieving web content in response to a Claude user's request.",
      "sourceUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "sourceCheckedAt": "2026-08-13"
    },
    {
      "slug": "google-extended",
      "token": "Google-Extended",
      "operator": "Google",
      "purpose": "mixed",
      "isFetcher": false,
      "description": "Google's robots.txt control token for Gemini model training and grounding uses of content already crawled by Google.",
      "sourceUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
      "sourceCheckedAt": "2026-08-13",
      "notes": "Control token only; Google says it is not a separate HTTP fetcher and does not affect Google Search inclusion or ranking."
    },
    {
      "slug": "applebot-extended",
      "token": "Applebot-Extended",
      "operator": "Apple",
      "purpose": "training",
      "isFetcher": false,
      "description": "Apple's robots.txt control token for opting content out of foundation-model training while leaving Applebot discovery separate.",
      "sourceUrl": "https://support.apple.com/119829",
      "sourceCheckedAt": "2026-08-13",
      "notes": "Control token applied to content fetched by Applebot."
    },
    {
      "slug": "perplexitybot",
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "purpose": "search",
      "isFetcher": true,
      "description": "Perplexity's automatic search crawler for surfacing and linking websites in Perplexity search results.",
      "sourceUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "sourceCheckedAt": "2026-08-13"
    },
    {
      "slug": "perplexity-user",
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "purpose": "user_action",
      "isFetcher": true,
      "description": "Perplexity's user-initiated fetcher for visiting pages while answering a user's question.",
      "sourceUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "sourceCheckedAt": "2026-08-13",
      "notes": "Perplexity says this user-triggered fetcher generally ignores robots.txt."
    },
    {
      "slug": "ccbot",
      "token": "CCBot",
      "operator": "Common Crawl",
      "purpose": "archive",
      "isFetcher": true,
      "description": "Common Crawl's automatic crawler for building its openly accessible web crawl repository.",
      "sourceUrl": "https://commoncrawl.org/ccbot",
      "sourceCheckedAt": "2026-08-13"
    }
  ]
}