{ "domain": "commoncrawl.org", "recordsNotConclusions": true, "events": [ { "id": "464b1ef9-143a-49dc-9c86-fba46fc71362", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-11T08:17:05.209Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Wed, 11 Nov 2026 08:17:05 GMT", "etag": "W/\"9175dcc6d4f929615b34ecee76d41e70\"", "last-modified": "Thu, 05 Nov 2026 22:25:51 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "c56033ad28065d877c2d04775f80e45700a8cb6ab7d9d8f77ba2a40e80b56f9d", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "1b29a33c-a104-4286-9662-4679abda2612", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "17e6fa8c0869a41f54a3ea46506bdff63149ac8b08a12721016f839541d7906a", "previousChainHash": "f9394a24136560370fa7bf7ee5b921057444913e8a8512a9885ff2d4fd531335" }, { "id": "f8c0ef8e-5b5f-49fd-a6b0-7054bde9312f", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-11T08:17:02.411Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Wed, 11 Nov 2026 08:17:02 GMT", "last-modified": "Tue, 10 Nov 2026 23:26:16 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "062aeb79b39a245139ee2142c35ec68fad0b149b42d889307f932c466845e581", "previousChainHash": "31f1bcd951722203e66236fec704eb09dabaee851ea724359fef6e839a4126f9" }, { "id": "d50f1c92-b5ce-4e1d-b466-ea315dad2f27", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-10T08:17:05.510Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Tue, 10 Nov 2026 08:17:05 GMT", "etag": "W/\"9175dcc6d4f929615b34ecee76d41e70\"", "last-modified": "Thu, 05 Nov 2026 22:25:51 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "c56033ad28065d877c2d04775f80e45700a8cb6ab7d9d8f77ba2a40e80b56f9d", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "1b29a33c-a104-4286-9662-4679abda2612", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f9394a24136560370fa7bf7ee5b921057444913e8a8512a9885ff2d4fd531335", "previousChainHash": "b910d5ac19d607cca5dc96caaf526430d097f6e3c5db0e4e6976dc629383b0d4" }, { "id": "28dd2ace-8b08-4266-b4c9-b549683a7eaf", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-10T08:17:02.680Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Tue, 10 Nov 2026 08:17:02 GMT", "last-modified": "Tue, 10 Nov 2026 01:27:44 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "31f1bcd951722203e66236fec704eb09dabaee851ea724359fef6e839a4126f9", "previousChainHash": "4b5bc72c37fff7724be72daef210601eaa1fe67222c70613f044b3cb6975bb37" }, { "id": "cdd75e9c-a4b6-491f-8edd-e86d23121b9b", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-09T08:17:05.168Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Mon, 09 Nov 2026 08:17:05 GMT", "etag": "W/\"9175dcc6d4f929615b34ecee76d41e70\"", "last-modified": "Thu, 05 Nov 2026 22:25:51 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "c56033ad28065d877c2d04775f80e45700a8cb6ab7d9d8f77ba2a40e80b56f9d", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "1b29a33c-a104-4286-9662-4679abda2612", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "b910d5ac19d607cca5dc96caaf526430d097f6e3c5db0e4e6976dc629383b0d4", "previousChainHash": "ea119e3c55b3864d77df23376a564e8dcf8f093b218e9c678dd7baa7f9e5324f" }, { "id": "b6a75a2f-82ae-409e-9be1-4e5f98a27b5a", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-09T08:17:02.151Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Mon, 09 Nov 2026 08:17:02 GMT", "last-modified": "Mon, 09 Nov 2026 00:06:05 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "4b5bc72c37fff7724be72daef210601eaa1fe67222c70613f044b3cb6975bb37", "previousChainHash": "4998f209780d21b9b6fd6348545385dfaf4051c565ae6d17a6e172683b3e02ea" }, { "id": "5fb3ee5c-7054-41f3-985a-286478b61bcf", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-08T08:17:05.002Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sun, 08 Nov 2026 08:17:05 GMT", "etag": "W/\"9175dcc6d4f929615b34ecee76d41e70\"", "last-modified": "Thu, 05 Nov 2026 22:25:51 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "c56033ad28065d877c2d04775f80e45700a8cb6ab7d9d8f77ba2a40e80b56f9d", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "1b29a33c-a104-4286-9662-4679abda2612", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ea119e3c55b3864d77df23376a564e8dcf8f093b218e9c678dd7baa7f9e5324f", "previousChainHash": "9ecb21b1c66418ecb7abcd5ea76b7c49badfeccfd0095beaa6f68913b05ef415" }, { "id": "1cd90b0f-45be-4e5b-8ebf-7722baa58418", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-08T08:17:02.104Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sun, 08 Nov 2026 08:17:02 GMT", "last-modified": "Sun, 08 Nov 2026 00:57:08 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "4998f209780d21b9b6fd6348545385dfaf4051c565ae6d17a6e172683b3e02ea", "previousChainHash": "1319c8f8d7ca7dc90dcab3054a2b04c134877c8ef7459ca29f65467c8d52e21e" }, { "id": "9f983951-dc88-433a-957d-c9d4022e4e43", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-07T08:17:05.159Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sat, 07 Nov 2026 08:17:05 GMT", "etag": "W/\"9175dcc6d4f929615b34ecee76d41e70\"", "last-modified": "Thu, 05 Nov 2026 22:25:51 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "c56033ad28065d877c2d04775f80e45700a8cb6ab7d9d8f77ba2a40e80b56f9d", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "1b29a33c-a104-4286-9662-4679abda2612", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9ecb21b1c66418ecb7abcd5ea76b7c49badfeccfd0095beaa6f68913b05ef415", "previousChainHash": "ff4b259500a70d512be752ec00e4dd4be0faa8a16dac7b32dbe4ec963f7d1f51" }, { "id": "9056b726-5dd8-4fcb-813f-9baa8994b20e", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-07T08:17:02.163Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sat, 07 Nov 2026 08:17:02 GMT", "last-modified": "Fri, 06 Nov 2026 22:32:25 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1319c8f8d7ca7dc90dcab3054a2b04c134877c8ef7459ca29f65467c8d52e21e", "previousChainHash": "8967c38fe67d71e28c80f2b1cccdf58a0a63267210d66501011fab86b5f98a91" }, { "id": "1b29a33c-a104-4286-9662-4679abda2612", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-06T08:17:05.030Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Fri, 06 Nov 2026 08:17:05 GMT", "etag": "W/\"9175dcc6d4f929615b34ecee76d41e70\"", "last-modified": "Thu, 05 Nov 2026 22:25:51 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "c56033ad28065d877c2d04775f80e45700a8cb6ab7d9d8f77ba2a40e80b56f9d", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "1b29a33c-a104-4286-9662-4679abda2612", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ff4b259500a70d512be752ec00e4dd4be0faa8a16dac7b32dbe4ec963f7d1f51", "previousChainHash": "83233e6d2f0a9d7fa008c077a01b1e473dba1d9667c461ac12110d2254350c8b" }, { "id": "51f2bfb7-0f40-4411-a73b-c83ebf120178", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-06T08:17:02.139Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Fri, 06 Nov 2026 08:17:02 GMT", "last-modified": "Thu, 05 Nov 2026 22:45:24 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8967c38fe67d71e28c80f2b1cccdf58a0a63267210d66501011fab86b5f98a91", "previousChainHash": "9056ada7017caf2c71f8a74dc7bd067ecc466c3961957d52203f263160c77162" }, { "id": "505ad14b-4568-44b2-afc1-8ba17bf25f82", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-05T08:17:05.691Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Thu, 05 Nov 2026 08:17:05 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "83233e6d2f0a9d7fa008c077a01b1e473dba1d9667c461ac12110d2254350c8b", "previousChainHash": "fdf462630b9267f66b72beebfc415113d9685a7ed9f435f35c1c6e81c1ffb6ea" }, { "id": "d6b670cf-c170-412f-a9e7-5aeec1dea9e4", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-05T08:17:02.769Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Thu, 05 Nov 2026 08:17:02 GMT", "last-modified": "Wed, 04 Nov 2026 23:16:53 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9056ada7017caf2c71f8a74dc7bd067ecc466c3961957d52203f263160c77162", "previousChainHash": "91b84e8f8e14ab995d486acc91fbbb7e999a01e2ff7b498faa7456993c00db22" }, { "id": "5f37ad8e-46a0-49ab-8d11-59f6296f9d91", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-04T08:17:05.559Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Wed, 04 Nov 2026 08:17:05 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "fdf462630b9267f66b72beebfc415113d9685a7ed9f435f35c1c6e81c1ffb6ea", "previousChainHash": "2a61caa953febceac5b9f0e1d7f2ed9babfe714dd5dd56432645d12c89f03deb" }, { "id": "91df9ad4-75ff-419d-9f83-cf6b5f02e2be", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-04T08:17:02.697Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Wed, 04 Nov 2026 08:17:02 GMT", "last-modified": "Tue, 03 Nov 2026 23:04:21 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "91b84e8f8e14ab995d486acc91fbbb7e999a01e2ff7b498faa7456993c00db22", "previousChainHash": "9ee5d7e49fdfe420d89770b9e036e621997d9e806bea7fd66341b6cbcc12d59a" }, { "id": "fbcaa849-865c-4f1a-9751-60db65aa83a8", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-03T08:17:06.111Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Tue, 03 Nov 2026 08:17:06 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2a61caa953febceac5b9f0e1d7f2ed9babfe714dd5dd56432645d12c89f03deb", "previousChainHash": "e2d2ef767018bb57735911d4c1a245cf38162041f8bbb85cfe27626c98038af8" }, { "id": "cd05eb4b-6581-4461-83e3-090c7b15c70d", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-03T08:17:03.166Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Tue, 03 Nov 2026 08:17:03 GMT", "last-modified": "Mon, 02 Nov 2026 22:37:35 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9ee5d7e49fdfe420d89770b9e036e621997d9e806bea7fd66341b6cbcc12d59a", "previousChainHash": "e7e865146bc121211b6dba805fa7177928e0e457e7679345cf21aba28d3a99b7" }, { "id": "87dbd4c7-09fe-4860-9417-e06f3861f900", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-02T08:17:05.590Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Mon, 02 Nov 2026 08:17:05 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e2d2ef767018bb57735911d4c1a245cf38162041f8bbb85cfe27626c98038af8", "previousChainHash": "22e26ca8eab07f315c3dceb9901e3fd4ca8e886d07e3f325b6141ae78081a2aa" }, { "id": "a9292944-0b79-47ec-8771-1ebf0c8fdd12", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-02T08:17:02.694Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Mon, 02 Nov 2026 08:17:02 GMT", "last-modified": "Mon, 02 Nov 2026 04:34:15 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e7e865146bc121211b6dba805fa7177928e0e457e7679345cf21aba28d3a99b7", "previousChainHash": "411822c4f5c04d730a16e2e792219d0d51d6316c62294d716e0d362f6482c5f8" }, { "id": "d8d19ced-342d-41df-8127-434f2be56a60", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-11-01T08:17:05.117Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sun, 01 Nov 2026 08:17:05 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "22e26ca8eab07f315c3dceb9901e3fd4ca8e886d07e3f325b6141ae78081a2aa", "previousChainHash": "95b33e5eb3c64f3da806ef3db0e20cac3fd1df5da57101528c23f0b6b1b0281d" }, { "id": "801c9539-856d-421b-a830-1a545f430053", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-11-01T08:17:02.193Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sun, 01 Nov 2026 08:17:02 GMT", "last-modified": "Sat, 31 Oct 2026 23:48:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "411822c4f5c04d730a16e2e792219d0d51d6316c62294d716e0d362f6482c5f8", "previousChainHash": "e66b1aaf293ceb94782b71101ba27b88b3142504598af17f6fffd953e3f5ec1e" }, { "id": "aaf8c2b0-0285-45a7-aefc-c5e928d35d0d", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-31T08:17:06.491Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sat, 31 Oct 2026 08:17:06 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "95b33e5eb3c64f3da806ef3db0e20cac3fd1df5da57101528c23f0b6b1b0281d", "previousChainHash": "f7852b6973c98ec607a2d25210c36bd7ecea80c47acbdcc672a3bcb539b4ebac" }, { "id": "2497fc87-3084-4eb6-a54c-4f89e07952c3", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-31T08:17:03.513Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sat, 31 Oct 2026 08:17:03 GMT", "last-modified": "Sat, 31 Oct 2026 01:11:57 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e66b1aaf293ceb94782b71101ba27b88b3142504598af17f6fffd953e3f5ec1e", "previousChainHash": "fe633c1b04becc047ba1814ac52ce13cac09f01d90fccf0f70b6d306da3963d9" }, { "id": "d1006ec9-6ef3-4714-8508-36df42ce2449", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-30T08:17:06.955Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Fri, 30 Oct 2026 08:17:06 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f7852b6973c98ec607a2d25210c36bd7ecea80c47acbdcc672a3bcb539b4ebac", "previousChainHash": "0135d088df9fa3bb2f164c5da4eb9386ead2073bf640806f138728d5e926158a" }, { "id": "a8c40df5-abcd-4511-80e8-7b46a834cad5", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-30T08:17:03.869Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Fri, 30 Oct 2026 08:17:03 GMT", "last-modified": "Fri, 30 Oct 2026 07:26:30 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "fe633c1b04becc047ba1814ac52ce13cac09f01d90fccf0f70b6d306da3963d9", "previousChainHash": "641181e3d8e017f112b4292b6671c6eec73bcaf698e090b34cc28960fa73e347" }, { "id": "6491317e-4b86-4c5d-83ec-e0b86f91a120", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-29T08:17:09.759Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Thu, 29 Oct 2026 08:17:09 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "0135d088df9fa3bb2f164c5da4eb9386ead2073bf640806f138728d5e926158a", "previousChainHash": "c0baf7246b01227d07d923cb7bf0413a83e9867879e1fb0d23c6540a13e2bdd5" }, { "id": "02a21dff-0fc1-4b86-b0a5-ad4f06594485", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-29T08:17:06.687Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Thu, 29 Oct 2026 08:17:06 GMT", "last-modified": "Wed, 28 Oct 2026 22:10:32 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "641181e3d8e017f112b4292b6671c6eec73bcaf698e090b34cc28960fa73e347", "previousChainHash": "e2c27547ad832a7e01eea244d5debd3db57d5ba19047ba12c2add279470dd6ab" }, { "id": "d813324d-b082-4289-a6ac-b85665a1fbf9", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-28T08:17:06.776Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Wed, 28 Oct 2026 08:17:06 GMT", "etag": "W/\"dddec5f0d57c7cbf3a6c143456448f4e\"", "last-modified": "Tue, 27 Oct 2026 09:46:31 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "5c757b02c8d4318d8f11cd3c60b3729243245e9796cf704ca54798312230ac94", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d813324d-b082-4289-a6ac-b85665a1fbf9", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c0baf7246b01227d07d923cb7bf0413a83e9867879e1fb0d23c6540a13e2bdd5", "previousChainHash": "768cb0f10356b4fa2dedcbc206c136ec8f31c2e8fceca789938f029bcf26ecae" }, { "id": "4211455e-828f-4c13-be2a-5d52be5e5c26", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-28T08:17:03.637Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Wed, 28 Oct 2026 08:17:03 GMT", "last-modified": "Tue, 27 Oct 2026 22:09:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e2c27547ad832a7e01eea244d5debd3db57d5ba19047ba12c2add279470dd6ab", "previousChainHash": "c38908e0d207566673aa470141042c38d17db84150e37b033f672a6c8bbcba48" }, { "id": "ea3c911b-bd61-4d5f-a067-be523c7d4516", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-27T08:17:05.809Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Tue, 27 Oct 2026 08:17:05 GMT", "etag": "W/\"22b6950fd5d05d4e49e9c45bb6ae7a62\"", "last-modified": "Thu, 22 Oct 2026 19:30:05 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "451d735e7564e80d5f9f2e64220508cff8f361e6c63ae2e22f4f83b68048fdf4", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "42bcef25-9218-4e06-925d-01c87b487f37", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "768cb0f10356b4fa2dedcbc206c136ec8f31c2e8fceca789938f029bcf26ecae", "previousChainHash": "ea5ece5489d92b8645955a89bf841baddac264d51029ba1f64aceba0c0d42d08" }, { "id": "3414ae34-482d-405f-99ab-3b6260bab7ac", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-27T08:17:02.296Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Tue, 27 Oct 2026 08:17:02 GMT", "last-modified": "Tue, 27 Oct 2026 07:45:17 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c38908e0d207566673aa470141042c38d17db84150e37b033f672a6c8bbcba48", "previousChainHash": "a7f8e6122763218f77b110a0cb66a493275246666def247abc96033910438aaf" }, { "id": "4999df1c-73ee-4749-bae4-d48d31ef236f", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-26T08:17:05.915Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Mon, 26 Oct 2026 08:17:05 GMT", "etag": "W/\"22b6950fd5d05d4e49e9c45bb6ae7a62\"", "last-modified": "Thu, 22 Oct 2026 19:30:05 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "451d735e7564e80d5f9f2e64220508cff8f361e6c63ae2e22f4f83b68048fdf4", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "42bcef25-9218-4e06-925d-01c87b487f37", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ea5ece5489d92b8645955a89bf841baddac264d51029ba1f64aceba0c0d42d08", "previousChainHash": "3ff1785780695c3872593ce432b1036fad343766075325bbe7447ade046619a7" }, { "id": "965940d2-d75d-4558-9a22-de8f963854d6", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-26T08:17:02.893Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Mon, 26 Oct 2026 08:17:02 GMT", "last-modified": "Mon, 26 Oct 2026 07:56:36 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a7f8e6122763218f77b110a0cb66a493275246666def247abc96033910438aaf", "previousChainHash": "c2057d5113bfb1fca34bd0841502760f5899228d74690dd5cb5744e4116e8699" }, { "id": "e25560b6-4424-4e3c-a218-4f1a5287df76", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-25T08:17:05.628Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sun, 25 Oct 2026 08:17:05 GMT", "etag": "W/\"22b6950fd5d05d4e49e9c45bb6ae7a62\"", "last-modified": "Thu, 22 Oct 2026 19:30:05 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "451d735e7564e80d5f9f2e64220508cff8f361e6c63ae2e22f4f83b68048fdf4", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "42bcef25-9218-4e06-925d-01c87b487f37", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3ff1785780695c3872593ce432b1036fad343766075325bbe7447ade046619a7", "previousChainHash": "7067419187dc953c613a75ca4eb9e555f9ce8fc6860097952fc73dcbb48e103e" }, { "id": "c739f204-a2ec-4b29-b436-03d4f4c88c67", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-25T08:17:02.664Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sun, 25 Oct 2026 08:17:02 GMT", "last-modified": "Sun, 25 Oct 2026 07:41:11 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c2057d5113bfb1fca34bd0841502760f5899228d74690dd5cb5744e4116e8699", "previousChainHash": "8aeb0453e9504c6bdf37b0cfde541d241bfec27ab7d25148b28924f3cb9e4fe6" }, { "id": "4805e839-9bcb-48fa-ace0-a07fb1ad07fd", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-24T08:17:04.839Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sat, 24 Oct 2026 08:17:04 GMT", "etag": "W/\"22b6950fd5d05d4e49e9c45bb6ae7a62\"", "last-modified": "Thu, 22 Oct 2026 19:30:05 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "451d735e7564e80d5f9f2e64220508cff8f361e6c63ae2e22f4f83b68048fdf4", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "42bcef25-9218-4e06-925d-01c87b487f37", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7067419187dc953c613a75ca4eb9e555f9ce8fc6860097952fc73dcbb48e103e", "previousChainHash": "713359435b7531f8af394b5ee434102cd6bd5bcf0be503fddfaab87642da9984" }, { "id": "955d4952-b44d-42ab-af6f-1f810a137ccc", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-24T08:17:02.052Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sat, 24 Oct 2026 08:17:02 GMT", "last-modified": "Sat, 24 Oct 2026 07:51:43 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8aeb0453e9504c6bdf37b0cfde541d241bfec27ab7d25148b28924f3cb9e4fe6", "previousChainHash": "5f84522c13d67c9257c73654c7a75b623ba83717a23c7d8d3998897c9ce6c36e" }, { "id": "42bcef25-9218-4e06-925d-01c87b487f37", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-23T08:17:08.897Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Fri, 23 Oct 2026 08:17:08 GMT", "etag": "W/\"22b6950fd5d05d4e49e9c45bb6ae7a62\"", "last-modified": "Thu, 22 Oct 2026 19:30:05 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "451d735e7564e80d5f9f2e64220508cff8f361e6c63ae2e22f4f83b68048fdf4", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "42bcef25-9218-4e06-925d-01c87b487f37", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "713359435b7531f8af394b5ee434102cd6bd5bcf0be503fddfaab87642da9984", "previousChainHash": "2bf8ddd9751043f8918259169ed925d5382497943bd42f5be815de36e2f5dbf7" }, { "id": "8c185059-b42f-47e0-a776-1837d02e963e", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-23T08:17:03.794Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Fri, 23 Oct 2026 08:17:03 GMT", "last-modified": "Fri, 23 Oct 2026 07:39:22 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "5f84522c13d67c9257c73654c7a75b623ba83717a23c7d8d3998897c9ce6c36e", "previousChainHash": "e1871ef9f1de521405f56eae3e439710ee331c719e2d1646e44e6c32c4c0e0d9" }, { "id": "cc683872-a653-4763-98b0-2d77f94da398", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-22T08:17:06.475Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Thu, 22 Oct 2026 08:17:06 GMT", "etag": "W/\"e5fb9ffa64c17fab650b61952368585b\"", "last-modified": "Wed, 21 Oct 2026 11:48:11 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": true, "artifactSha256": "7ee253f82b9e752e9527bb40b8d1540f094942cf3da5bbc201226c95b53a1c34", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "cc683872-a653-4763-98b0-2d77f94da398", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2bf8ddd9751043f8918259169ed925d5382497943bd42f5be815de36e2f5dbf7", "previousChainHash": "d7c5449aa86f622aa5ee02b6983a930928af820204794485c62f819e0dfe6d53" }, { "id": "cc3c01f5-a11e-4a9e-a47a-9c52c0f9bd9c", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-22T08:17:03.524Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Thu, 22 Oct 2026 08:17:03 GMT", "last-modified": "Thu, 22 Oct 2026 00:51:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e1871ef9f1de521405f56eae3e439710ee331c719e2d1646e44e6c32c4c0e0d9", "previousChainHash": "ff04d11f9ffeca2edfc151ba34cc3369ca9c059d2bcf1bbb8fa08bf088c15bb4" }, { "id": "8d98006c-66ec-4442-ab4d-5380624486ec", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-21T08:17:06.489Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Wed, 21 Oct 2026 08:17:06 GMT", "etag": "W/\"fb3b4e1a1c4e8d92421f2a35cd48e09a\"", "last-modified": "Mon, 12 Oct 2026 20:46:09 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "bfdf76d3b503fe78c29d91b9eba0aef375b9b15addd8996262cef598812aa7b9", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d6a8fc18-bbc6-4970-9ae6-5c23b4891ad3", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d7c5449aa86f622aa5ee02b6983a930928af820204794485c62f819e0dfe6d53", "previousChainHash": "24bf30b520ebd6df9987d6cbc4cda84fda8e04b880336abe51573508a2bcb12d" }, { "id": "cbaf4c31-74fc-441a-a250-faf9c881214b", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-21T08:17:03.612Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Wed, 21 Oct 2026 08:17:03 GMT", "last-modified": "Tue, 20 Oct 2026 21:22:06 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ff04d11f9ffeca2edfc151ba34cc3369ca9c059d2bcf1bbb8fa08bf088c15bb4", "previousChainHash": "e2cb47c626c16bf69c4cd3f54bcda24866bf449c3e0da1f3acb941e6de24e455" }, { "id": "9ece69fd-c732-4b3a-a8f8-0ce9c7487661", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-20T08:17:05.732Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Tue, 20 Oct 2026 08:17:05 GMT", "etag": "W/\"fb3b4e1a1c4e8d92421f2a35cd48e09a\"", "last-modified": "Mon, 12 Oct 2026 20:46:09 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "bfdf76d3b503fe78c29d91b9eba0aef375b9b15addd8996262cef598812aa7b9", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d6a8fc18-bbc6-4970-9ae6-5c23b4891ad3", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "24bf30b520ebd6df9987d6cbc4cda84fda8e04b880336abe51573508a2bcb12d", "previousChainHash": "f5a14ee05df14492d97de6c032df0cbd3f67536c7d492b6104f8cf2d2085b9b8" }, { "id": "672bfc85-385d-4e1e-9467-b5dd129409e3", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-20T08:17:02.555Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Tue, 20 Oct 2026 08:17:02 GMT", "last-modified": "Mon, 19 Oct 2026 21:18:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e2cb47c626c16bf69c4cd3f54bcda24866bf449c3e0da1f3acb941e6de24e455", "previousChainHash": "518b1a2622e44691c3a92ad2704801479bf78b2f557576216c1e4008ca93009c" }, { "id": "7ec07876-f42a-4483-88f4-b986dc136fa7", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-19T08:17:05.871Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Mon, 19 Oct 2026 08:17:05 GMT", "etag": "W/\"fb3b4e1a1c4e8d92421f2a35cd48e09a\"", "last-modified": "Mon, 12 Oct 2026 20:46:09 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "bfdf76d3b503fe78c29d91b9eba0aef375b9b15addd8996262cef598812aa7b9", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d6a8fc18-bbc6-4970-9ae6-5c23b4891ad3", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f5a14ee05df14492d97de6c032df0cbd3f67536c7d492b6104f8cf2d2085b9b8", "previousChainHash": "2c991f9b1670376446092e837571206eabd4e694a622681af7c683b2778bcf7a" }, { "id": "52c82a8b-cdd7-4ac8-8938-e40aafa431e8", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-19T08:17:02.911Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Mon, 19 Oct 2026 08:17:02 GMT", "last-modified": "Sun, 18 Oct 2026 23:23:52 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "518b1a2622e44691c3a92ad2704801479bf78b2f557576216c1e4008ca93009c", "previousChainHash": "3a48b0e458f7fe8e3e03d8274d7dca3171b7bab76614f4453b384ce158696c5e" }, { "id": "a210545c-1d65-4d20-9cca-3cdf24a70b06", "domain": "commoncrawl.org", "signal": "llms", "requestedUrl": "https://commoncrawl.org/llms.txt", "finalUrl": "https://commoncrawl.org/llms.txt", "fetchedAt": "2026-10-18T08:17:06.460Z", "httpStatus": 404, "responseHeaders": { "cache-control": "max-age=43200", "content-type": "text/html; charset=utf-8", "date": "Sun, 18 Oct 2026 08:17:06 GMT", "etag": "W/\"fb3b4e1a1c4e8d92421f2a35cd48e09a\"", "last-modified": "Mon, 12 Oct 2026 20:46:09 GMT", "server": "cloudflare" }, "parseStatus": "absent", "changed": false, "artifactSha256": "bfdf76d3b503fe78c29d91b9eba0aef375b9b15addd8996262cef598812aa7b9", "artifactBytes": 4558, "parserVersion": null, "parsed": null, "errorCode": null, "canonicalEventId": "d6a8fc18-bbc6-4970-9ae6-5c23b4891ad3", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2c991f9b1670376446092e837571206eabd4e694a622681af7c683b2778bcf7a", "previousChainHash": "102617d4f4a4e4493f0dfad854697840f4765215747f375200541c47b1c8cc94" }, { "id": "562c0120-b99e-4baa-873a-b9450ef05346", "domain": "commoncrawl.org", "signal": "robots", "requestedUrl": "https://commoncrawl.org/robots.txt", "finalUrl": "https://commoncrawl.org/robots.txt", "fetchedAt": "2026-10-18T08:17:02.808Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain; charset=utf-8", "date": "Sun, 18 Oct 2026 08:17:02 GMT", "last-modified": "Sat, 17 Oct 2026 22:46:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "13ac8bbfcbc869e1eaa4b87e5a805cb83a9b7b6ff0a4471db382793e88263ebb", "artifactBytes": 4508, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "IndeedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" } ] }, { "agents": [ "Indeedshopbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 55, "pattern": "/" } ] }, { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://commoncrawl.org/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 58, "pattern": "/search?*" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4891637e-ebca-4070-b691-0d2a74bf7930", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3a48b0e458f7fe8e3e03d8274d7dca3171b7bab76614f4453b384ce158696c5e", "previousChainHash": "04c6368c4b9c927ebbdff4d0c8051dce35d5bef08ff43f60920485fbd5971702" } ] }