{
  "domain": "commoncrawl.org",
  "recordsNotConclusions": true,
  "observations": [
    {
      "id": "adad742f-d9e3-4863-9824-035eb497fe5f",
      "domain": "commoncrawl.org",
      "signal": "llms",
      "requestedUrl": "https://commoncrawl.org/llms.txt",
      "finalUrl": "https://commoncrawl.org/llms.txt",
      "fetchedAt": "2026-08-13T08:17:16.516Z",
      "httpStatus": 404,
      "responseHeaders": {
        "cache-control": "max-age=43200",
        "content-type": "text/html; charset=utf-8",
        "date": "Thu, 13 Aug 2026 08:17:16 GMT",
        "etag": "W/\"f69427f0dd75a9f8de6411bfa9f09f01\"",
        "last-modified": "Wed, 12 Aug 2026 08:46:12 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "absent",
      "changed": true,
      "artifactSha256": "c1b8467efd09d989fad0af9b96964a42135100011b15f81d0915737ecde558b0",
      "artifactBytes": 4558,
      "parserVersion": null,
      "parsed": null,
      "errorCode": null,
      "canonicalEventId": "adad742f-d9e3-4863-9824-035eb497fe5f",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "8784e1a32be3718dcd54ab4552d0384529785563d134abf8aeb8b39993988930",
      "previousChainHash": "4759f0527df0034fd9f72cbeb6f57312c10aa77f33c12e9c02d68648bcb8e112"
    },
    {
      "id": "b6b8f256-e2c5-4aea-8752-c40bbf60f6fa",
      "domain": "commoncrawl.org",
      "signal": "robots",
      "requestedUrl": "https://commoncrawl.org/robots.txt",
      "finalUrl": "https://commoncrawl.org/robots.txt",
      "fetchedAt": "2026-08-13T08:17:13.488Z",
      "httpStatus": 200,
      "responseHeaders": {
        "content-type": "text/plain; charset=utf-8",
        "date": "Thu, 13 Aug 2026 08:17:13 GMT",
        "last-modified": "Wed, 12 Aug 2026 21:19:35 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "parsed",
      "changed": false,
      "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907",
      "artifactBytes": 4452,
      "parserVersion": "robots-rfc9309-rsl-0.1.0",
      "parsed": {
        "globalLicenses": [],
        "groups": [
          {
            "agents": [
              "*"
            ],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ]
          }
        ],
        "parserVersion": "robots-rfc9309-rsl-0.1.0",
        "registryVersion": "2026-08-03",
        "sitemaps": [
          "https://commoncrawl.org/sitemap.xml"
        ],
        "trackedAgents": {
          "Applebot-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "CCBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "ChatGPT-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "Claude-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "Claude-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "ClaudeBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "Google-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "GPTBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "OAI-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "Perplexity-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          },
          "PerplexityBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 52,
                "pattern": "/"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/search?*"
              }
            ],
            "state": "partial"
          }
        },
        "warnings": []
      },
      "errorCode": null,
      "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "5942dbc4cca55a1d2b5c91d1c81a3ca3e5a30e6edae7d731ac8c0bd4a57b5de1",
      "previousChainHash": "ec1533760e488cd8027f048c1abdbfb092270e430b8a6c124fdf202d661b52a8"
    }
  ],
  "payPerCrawl": {
    "status": "unverified",
    "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API."
  }
}