{ "domain": "commoncrawl.org", "recordsNotConclusions": true, "events": [ { "id": "adad742f-d9e3-4863-9824-035eb497fe5f", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-13T08:17:16.516Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Thu, 13 Aug 2026 08:17:16 GMT", "etag": "W/\"f69427f0dd75a9f8de6411bfa9f09f01\"", "last-modified": "Wed, 12 Aug 2026 08:46:12 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "c1b8467efd09d989fad0af9b96964a42135100011b15f81d0915737ecde558b0", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "adad742f-d9e3-4863-9824-035eb497fe5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8784e1a32be3718dcd54ab4552d0384529785563d134abf8aeb8b39993988930", "previousChainHash": "4759f0527df0034fd9f72cbeb6f57312c10aa77f33c12e9c02d68648bcb8e112" }, { "id": "b6b8f256-e2c5-4aea-8752-c40bbf60f6fa", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-13T08:17:13.488Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Thu, 13 Aug 2026 08:17:13 GMT", "last-modified": "Wed, 12 Aug 2026 21:19:35 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "5942dbc4cca55a1d2b5c91d1c81a3ca3e5a30e6edae7d731ac8c0bd4a57b5de1", "previousChainHash": "ec1533760e488cd8027f048c1abdbfb092270e430b8a6c124fdf202d661b52a8" }, { "id": "99b620a7-c563-4a3a-9c67-24aa09fb5c3d", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-12T08:17:16.335Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Wed, 12 Aug 2026 08:17:16 GMT", "etag": "W/\"5e2652e605ea9433707026c9e9da332d\"", "last-modified": "Tue, 11 Aug 2026 19:44:28 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "ffa87ff7e775fab8ecf3dbac61012b00599a41bc174d77cb361216ce157e75d3", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "99b620a7-c563-4a3a-9c67-24aa09fb5c3d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "4759f0527df0034fd9f72cbeb6f57312c10aa77f33c12e9c02d68648bcb8e112", "previousChainHash": "ca4e39413e80c92fc10524d719dc4a4150e33525e12be2d6b3103383ec8ec965" }, { "id": "bd722a99-bf58-4565-8c8a-bca085b956aa", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-12T08:17:13.300Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Wed, 12 Aug 2026 08:17:13 GMT", "last-modified": "Wed, 12 Aug 2026 08:00:38 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ec1533760e488cd8027f048c1abdbfb092270e430b8a6c124fdf202d661b52a8", "previousChainHash": "6edb2f7965313fb405da00401911b57b8212fb59116f15c831bc2a13f592d2c9" }, { "id": "0cdecb8b-97fa-439f-ac68-a5d85856eb6f", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-11T08:17:16.091Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Tue, 11 Aug 2026 08:17:16 GMT", "etag": "W/\"c29dd5a528c28f75b9506b21899589e1\"", "last-modified": "Mon, 10 Aug 2026 15:54:56 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "9fd51c0ba05b6ad7260d325363f69834853ee1f6e806c659c17ac8e8517e3e76", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "0cdecb8b-97fa-439f-ac68-a5d85856eb6f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ca4e39413e80c92fc10524d719dc4a4150e33525e12be2d6b3103383ec8ec965", "previousChainHash": "cbb7d1011258e9423f416d5607b0974cea16c65556abe88c0df69b256e73f88d" }, { "id": "e21718eb-4f88-4165-918b-d7643c848b2b", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-11T08:17:13.108Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Tue, 11 Aug 2026 08:17:13 GMT", "last-modified": "Tue, 11 Aug 2026 04:13:57 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6edb2f7965313fb405da00401911b57b8212fb59116f15c831bc2a13f592d2c9", "previousChainHash": "728e5508078e74c64b9b8c50c50e4b9be3db6284c2f1eec995938430b57f1c34" }, { "id": "f18828e0-467a-4472-9145-569f8e4efe71", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-10T08:17:15.970Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Mon, 10 Aug 2026 08:17:15 GMT", "etag": "W/\"f8baf502f5519ef28c3deac19546c099\"", "last-modified": "Tue, 04 Aug 2026 18:56:39 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "b05bb082fa3762c1a624c6acb373371b6b9aa87d2e14d62f58919b4a81e31c90", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "97fb777a-0451-4e9b-85a6-6721a2893a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "cbb7d1011258e9423f416d5607b0974cea16c65556abe88c0df69b256e73f88d", "previousChainHash": "18e2c3cc62b321f776d38545849c6f4314c5ec7276d1fad7d993c4ec4458bd65" }, { "id": "07435968-0ead-4c3d-8ddc-2b328fec75a8", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-10T08:17:13.172Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Mon, 10 Aug 2026 08:17:13 GMT", "last-modified": "Mon, 10 Aug 2026 03:56:23 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "728e5508078e74c64b9b8c50c50e4b9be3db6284c2f1eec995938430b57f1c34", "previousChainHash": "32420643483bf9cb40784191bcf74ab7ae7216ad2cdf10b0a3384000987d3557" }, { "id": "d5f93ec6-c2ad-429e-afb8-4a7f111b0dd8", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-09T08:17:17.157Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sun, 09 Aug 2026 08:17:17 GMT", "etag": "W/\"f8baf502f5519ef28c3deac19546c099\"", "last-modified": "Tue, 04 Aug 2026 18:56:39 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "b05bb082fa3762c1a624c6acb373371b6b9aa87d2e14d62f58919b4a81e31c90", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "97fb777a-0451-4e9b-85a6-6721a2893a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "18e2c3cc62b321f776d38545849c6f4314c5ec7276d1fad7d993c4ec4458bd65", "previousChainHash": "c1a86b8e5eee170c970758260ea21380da0738ae87c85ed6ddfce96cff1512a4" }, { "id": "270b7542-e57d-4774-b67b-46c4d861f0c0", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-09T08:17:14.327Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sun, 09 Aug 2026 08:17:14 GMT", "last-modified": "Sun, 09 Aug 2026 08:17:14 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "32420643483bf9cb40784191bcf74ab7ae7216ad2cdf10b0a3384000987d3557", "previousChainHash": "d34bc6890b3963f444d29592078b1a9be04e86f550ece67a4bf257b843acf675" }, { "id": "bc5b51eb-fae1-4fdf-a10c-b19464f78acf", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-08T08:17:17.118Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sat, 08 Aug 2026 08:17:17 GMT", "etag": "W/\"f8baf502f5519ef28c3deac19546c099\"", "last-modified": "Tue, 04 Aug 2026 18:56:39 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "b05bb082fa3762c1a624c6acb373371b6b9aa87d2e14d62f58919b4a81e31c90", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "97fb777a-0451-4e9b-85a6-6721a2893a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c1a86b8e5eee170c970758260ea21380da0738ae87c85ed6ddfce96cff1512a4", "previousChainHash": "f857ad7db5e364a828bdfc96d7645dcde4352e940713edd5e8519bbd38a35283" }, { "id": "6e2525a5-6a02-4caa-a48a-d07d909b4399", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-08T08:17:14.075Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sat, 08 Aug 2026 08:17:14 GMT", "last-modified": "Sat, 08 Aug 2026 02:56:27 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d34bc6890b3963f444d29592078b1a9be04e86f550ece67a4bf257b843acf675", "previousChainHash": "0cf70401f79d3fc579582ff9f9afda860d6d4b0c807d74fa675ac2f84b5d1715" }, { "id": "43a8895f-45bc-465e-9981-b2dc0a15c2a1", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-07T08:17:16.813Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Fri, 07 Aug 2026 08:17:16 GMT", "etag": "W/\"f8baf502f5519ef28c3deac19546c099\"", "last-modified": "Tue, 04 Aug 2026 18:56:39 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "b05bb082fa3762c1a624c6acb373371b6b9aa87d2e14d62f58919b4a81e31c90", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "97fb777a-0451-4e9b-85a6-6721a2893a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f857ad7db5e364a828bdfc96d7645dcde4352e940713edd5e8519bbd38a35283", "previousChainHash": "f9daef5676a10b980556e5784136e7cf4a0caf64b7f0cd38f8c80a78abb6dedd" }, { "id": "28f50af5-376c-4d46-aaf4-fd32d7323e2d", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-07T08:17:13.740Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Fri, 07 Aug 2026 08:17:13 GMT", "last-modified": "Fri, 07 Aug 2026 07:19:09 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "0cf70401f79d3fc579582ff9f9afda860d6d4b0c807d74fa675ac2f84b5d1715", "previousChainHash": "372e58b53de976323a8ed3cd0cddc585f51e1ff692d765e59b9beb105d42795b" }, { "id": "66aae575-989a-4d5c-9ceb-825a8c33d3db", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-06T08:17:16.341Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Thu, 06 Aug 2026 08:17:16 GMT", "etag": "W/\"f8baf502f5519ef28c3deac19546c099\"", "last-modified": "Tue, 04 Aug 2026 18:56:39 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "b05bb082fa3762c1a624c6acb373371b6b9aa87d2e14d62f58919b4a81e31c90", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "97fb777a-0451-4e9b-85a6-6721a2893a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f9daef5676a10b980556e5784136e7cf4a0caf64b7f0cd38f8c80a78abb6dedd", "previousChainHash": "e29a67db7c86cb28d8ab09bae1d71ef8e5e8ac30ac9a25c39ea44021c124f9ef" }, { "id": "45f89378-2d78-4054-a0c1-4a97c16c4771", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-06T08:17:13.261Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Thu, 06 Aug 2026 08:17:13 GMT", "last-modified": "Thu, 06 Aug 2026 07:15:14 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "372e58b53de976323a8ed3cd0cddc585f51e1ff692d765e59b9beb105d42795b", "previousChainHash": "a6de50b24afd0999a61b268df8b75ba76454cb5d7a139a7bd5a4c4c36772404c" }, { "id": "97fb777a-0451-4e9b-85a6-6721a2893a50", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-05T08:17:15.973Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Wed, 05 Aug 2026 08:17:15 GMT", "etag": "W/\"f8baf502f5519ef28c3deac19546c099\"", "last-modified": "Tue, 04 Aug 2026 18:56:39 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "b05bb082fa3762c1a624c6acb373371b6b9aa87d2e14d62f58919b4a81e31c90", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "97fb777a-0451-4e9b-85a6-6721a2893a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e29a67db7c86cb28d8ab09bae1d71ef8e5e8ac30ac9a25c39ea44021c124f9ef", "previousChainHash": "83a30abd89c89d66e5f4f1fa4eabf660efe091caefa8afb4153446fec773bd5e" }, { "id": "2d279512-cfc9-48b5-8231-ce7e432d41a0", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-05T08:17:13.056Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Wed, 05 Aug 2026 08:17:13 GMT", "last-modified": "Wed, 05 Aug 2026 07:17:52 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a6de50b24afd0999a61b268df8b75ba76454cb5d7a139a7bd5a4c4c36772404c", "previousChainHash": "e239fac51a8092999250ccf46c6ed767f5dd3b1a6a1b815149e3a17c8bc14337" }, { "id": "7a49a4d0-1196-45fd-a5d5-3f2ad4e8c05b", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-08-04T08:17:25.953Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Tue, 04 Aug 2026 08:17:25 GMT", "etag": "W/\"5fc8037795dc661a833ba7f4be61342f\"", "last-modified": "Mon, 03 Aug 2026 20:50:58 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "91aa56a993d20c3531e6ffe4562bc9d244fdded984ccf3be0ec37568918df4ee", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "7a49a4d0-1196-45fd-a5d5-3f2ad4e8c05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "83a30abd89c89d66e5f4f1fa4eabf660efe091caefa8afb4153446fec773bd5e", "previousChainHash": null }, { "id": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-08-04T08:17:19.117Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Tue, 04 Aug 2026 08:17:19 GMT", "last-modified": "Tue, 04 Aug 2026 02:03:01 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "87e17d4f9c85b62e12eee2850cde9f9ec9336adf6106b482bcd32bd3806b3907", "artifactBytes": 4452, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 52, "pattern": "/" }, { "directive": "disallow", "line": 53, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "782bc5d7-12eb-4912-b397-9ccfddedf05b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e239fac51a8092999250ccf46c6ed767f5dd3b1a6a1b815149e3a17c8bc14337", "previousChainHash": null } ] }