{
  "recordsNotConclusions": true,
  "registryVersion": "2026-08-13",
  "agent": {
    "slug": "gptbot",
    "token": "GPTBot",
    "operator": "OpenAI",
    "purpose": "training",
    "isFetcher": true,
    "description": "OpenAI's automatic crawler for content that may be used to improve and train generative AI foundation models.",
    "sourceUrl": "https://developers.openai.com/api/docs/bots",
    "sourceCheckedAt": "2026-08-13"
  },
  "observations": [
    {
      "domain": "anthropic.com",
      "state": "allowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T04:17:13.780Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "apnews.com",
      "state": "disallowed",
      "matchedAgents": [
        "GPTBot"
      ],
      "lastObservedAt": "2026-08-13T15:17:13.151Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0"
    },
    {
      "domain": "apple.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T06:17:13.327Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "cloudflare.com",
      "state": "allowed",
      "matchedAgents": [
        "GPTBot"
      ],
      "lastObservedAt": "2026-08-13T02:17:13.781Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "commoncrawl.org",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T08:17:13.488Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "google.com",
      "state": "partial",
      "matchedAgents": [
        "*",
        "Yandex"
      ],
      "lastObservedAt": "2026-08-13T07:17:17.570Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "medium.com",
      "state": "disallowed",
      "matchedAgents": [
        "Amazonbot",
        "Applebot-Extended",
        "Bytespider",
        "ClaudeBot",
        "FacebookBot",
        "GoogleOther",
        "GPTBot",
        "meta-externalagent"
      ],
      "lastObservedAt": "2026-08-13T11:17:13.383Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "openai.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T03:17:16.191Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "oreilly.com",
      "state": "disallowed",
      "matchedAgents": [
        "GPTBot",
        "anthropic-ai",
        "Google-Extended",
        "cohere-ai",
        "CCBot",
        "AI2Bot",
        "Amazonbot",
        "Amazonbot-Video",
        "Bytespider",
        "meta-externalagent",
        "Diffbot",
        "omgili",
        "TimpiBot",
        "SeznamBot",
        "Exabot",
        "YandexBot",
        "Sogou",
        "360Spider",
        "YisouSpider",
        "Baiduspider-Render/2.0",
        "MJ12bot",
        "DataForSeoBot",
        "AliyunSecBot",
        "ArchiveTeam",
        "ArchiveTeam ArchiveBot",
        "AwarioBot",
        "ZoominfoBot",
        "ali-implementer",
        "Blueno",
        "BIGO-baiguoyuan",
        "WorksOgCrawler",
        "OnPageBot",
        "DoCoMo",
        "SAMSUNG-SGH-E250",
        "TA-Googlebot"
      ],
      "lastObservedAt": "2026-08-13T13:17:13.177Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "perplexity.ai",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T05:17:13.768Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "quora.com",
      "state": "disallowed",
      "matchedAgents": [
        "GPTBot"
      ],
      "lastObservedAt": "2026-08-13T12:17:13.617Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "reddit.com",
      "state": "disallowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T09:17:13.755Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "rslcollective.org",
      "state": "allowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T01:17:13.460Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "rslstandard.org",
      "state": "allowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T00:17:13.631Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "stackoverflow.com",
      "state": "unverified",
      "matchedAgents": [],
      "lastObservedAt": "2026-08-12T17:17:13.588Z",
      "parserVersion": null
    },
    {
      "domain": "theguardian.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T16:17:14.036Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0"
    },
    {
      "domain": "usatoday.com",
      "state": "disallowed",
      "matchedAgents": [
        "GPTBot"
      ],
      "lastObservedAt": "2026-08-12T19:17:13.079Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "voxmedia.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-12T18:17:14.665Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "yahoo.com",
      "state": "disallowed",
      "matchedAgents": [
        "ADmantX",
        "AlphaBot",
        "anthropic-ai",
        "AwarioRssBot",
        "AwarioSmartBot",
        "BLEXBot",
        "Buzzbot",
        "Bytespider",
        "CCBot",
        "ChatGPT-User",
        "claritybot",
        "Claude-Web",
        "ClaudeBot",
        "cohere-ai",
        "Diffbot",
        "FacebookBot",
        "FriendlyCrawler",
        "Google-Extended",
        "GPTBot",
        "huggingface",
        "ImagesiftBot",
        "img2dataset",
        "magpie-crawler",
        "Meltwater",
        "Neevabot",
        "news-please",
        "NewsNow",
        "Nutch",
        "omgili",
        "omgilibot",
        "panscient.com",
        "Perplexity-ai",
        "PerplexityBot",
        "PetalBot",
        "PiplBot",
        "scoop.it",
        "Scrapy",
        "Seekr",
        "SentiBot",
        "SeznamBot",
        "TurnitinBot",
        "YouBot",
        "ZumBot"
      ],
      "lastObservedAt": "2026-08-13T10:17:13.627Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    },
    {
      "domain": "ziffdavis.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-08-13T14:17:13.413Z",
      "parserVersion": "robots-rfc9309-rsl-0.1.0"
    }
  ]
}