{
  "domain": "openai.com",
  "recordsNotConclusions": true,
  "observations": [
    {
      "id": "130f9a3e-897f-4b93-aa31-5d032f0b20aa",
      "domain": "openai.com",
      "signal": "llms",
      "requestedUrl": "https://openai.com/llms.txt",
      "finalUrl": "https://openai.com/llms.txt",
      "fetchedAt": "2026-08-13T03:17:22.800Z",
      "httpStatus": 403,
      "responseHeaders": {
        "content-length": "9853",
        "content-type": "text/html; charset=UTF-8",
        "date": "Thu, 13 Aug 2026 03:17:22 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "http_error",
      "changed": true,
      "artifactSha256": "984bbb506f763008a139ba5b0bda50bfbbc28d23f6609c1308567cfe9a668c84",
      "artifactBytes": 9853,
      "parserVersion": null,
      "parsed": null,
      "errorCode": "http_403",
      "canonicalEventId": "130f9a3e-897f-4b93-aa31-5d032f0b20aa",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "5d2bb143ff282f6b3f4df7ee40336783bc6759cf325685b540d2fa870225de13",
      "previousChainHash": "4fbc67ad3d3eac1024a7c2c2790242dd08cf1f83d21d8921999564d6960b985b"
    },
    {
      "id": "7b7521ed-68ec-4e86-9b99-974ebc1fb106",
      "domain": "openai.com",
      "signal": "robots",
      "requestedUrl": "https://openai.com/robots.txt",
      "finalUrl": "https://openai.com/robots.txt",
      "fetchedAt": "2026-08-13T03:17:16.191Z",
      "httpStatus": 304,
      "responseHeaders": {
        "cache-control": "public, max-age=0, must-revalidate",
        "date": "Thu, 13 Aug 2026 03:17:16 GMT",
        "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"",
        "last-modified": "Thu, 16 Apr 2026 00:44:06 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "unchanged",
      "changed": false,
      "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498",
      "artifactBytes": 98,
      "parserVersion": "robots-rfc9309-rsl-0.1.0",
      "parsed": {
        "globalLicenses": [],
        "groups": [
          {
            "agents": [
              "*"
            ],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ]
          }
        ],
        "parserVersion": "robots-rfc9309-rsl-0.1.0",
        "registryVersion": "2026-08-03",
        "sitemaps": [
          "https://openai.com/sitemap.xml"
        ],
        "trackedAgents": {
          "Applebot-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "CCBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "ChatGPT-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "Claude-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "Claude-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "ClaudeBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "Google-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "GPTBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "OAI-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "Perplexity-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          },
          "PerplexityBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 2,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/microsoft-for-startups/"
              }
            ],
            "state": "partial"
          }
        },
        "warnings": []
      },
      "errorCode": null,
      "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "6b0916897f05c0a58ee73ad8d6678c15ee71502b45634deff817b92848ec77dc",
      "previousChainHash": "bd4b26d4571eec4777a4c3a95a1dc35a52bd9aa0e74a9c15a71b65aadaabf466"
    }
  ],
  "payPerCrawl": {
    "status": "unverified",
    "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API."
  }
}