{ "domain": "cloudflare.com", "recordsNotConclusions": true, "events": [ { "id": "0e6f4566-6a7d-4221-b77e-b27b7cf170aa", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-13T02:17:16.553Z", "httpStatus": 301, "responseHeaders": { "content-type": "text/html; charset=UTF-8", "date": "Thu, 13 Aug 2026 02:17:15 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "0e6f4566-6a7d-4221-b77e-b27b7cf170aa", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "5cbc4369a3a3eb7f80f8829d18d8cd30dfe696e92d94c9d38fa6543221feb305", "previousChainHash": "88062becdac35abd97e88523dc613aa61d252bff7705c536ac1232955063f3c2" }, { "id": "e1ad94ab-b958-418a-aa0f-4a8fdb823c86", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-13T02:17:13.781Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Thu, 13 Aug 2026 02:17:13 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bdb0d7c228882849831200cc2f7651a18151166288b3702ee462a3034ab95dac", "previousChainHash": "a0f91d4222d39192ee158e7e7ef4c3f24068266e1e2d677862b1e4624f1f5237" }, { "id": "b0693677-6f3b-4e3a-893d-4b3322b2e643", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-12T02:17:16.960Z", "httpStatus": 301, "responseHeaders": { "content-type": "text/html; charset=UTF-8", "date": "Wed, 12 Aug 2026 02:17:16 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "b0693677-6f3b-4e3a-893d-4b3322b2e643", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "88062becdac35abd97e88523dc613aa61d252bff7705c536ac1232955063f3c2", "previousChainHash": "c4a87158062a89e24736ac7adb583940120f73d041d222038e5321e9fbeac827" }, { "id": "5b234cb9-0d19-4359-a137-c66a534bf037", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-12T02:17:14.347Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Wed, 12 Aug 2026 02:17:14 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a0f91d4222d39192ee158e7e7ef4c3f24068266e1e2d677862b1e4624f1f5237", "previousChainHash": "9c297c5d8ba68c70b341b3bc20348ff5c76cabc343815819eca4b72a395d1797" }, { "id": "8214d312-1b7f-4f5b-87be-1682b0780527", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-11T02:17:17.331Z", "httpStatus": 301, "responseHeaders": { "content-type": "text/html; charset=UTF-8", "date": "Tue, 11 Aug 2026 02:17:16 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "8214d312-1b7f-4f5b-87be-1682b0780527", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c4a87158062a89e24736ac7adb583940120f73d041d222038e5321e9fbeac827", "previousChainHash": "394b6de396f350bd271b99f9ed2db7814d2daf9b004a5c9ad003f219a2f00fd4" }, { "id": "44ab8a14-c2ef-4bc5-9f58-523250064319", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-11T02:17:14.372Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Tue, 11 Aug 2026 02:17:13 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9c297c5d8ba68c70b341b3bc20348ff5c76cabc343815819eca4b72a395d1797", "previousChainHash": "7f4b14dd80254e6e9c6b8bb0e384b8da2dc7e86e7ad136567feb5e5eb5bf2dc0" }, { "id": "2a6c4bd4-298a-42e0-bf61-15779a966941", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-10T02:17:14.252Z", "httpStatus": 301, "responseHeaders": { "content-type": "text/html; charset=UTF-8", "date": "Mon, 10 Aug 2026 02:17:13 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "2a6c4bd4-298a-42e0-bf61-15779a966941", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "394b6de396f350bd271b99f9ed2db7814d2daf9b004a5c9ad003f219a2f00fd4", "previousChainHash": "8eddc56cd2477c36b0e7a7487e4bf90842acea0fb361ee7f685a2fa9e98d6def" }, { "id": "3f058501-14d2-4249-a08a-b75607e5c95d", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-10T02:17:11.479Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Mon, 10 Aug 2026 02:17:11 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7f4b14dd80254e6e9c6b8bb0e384b8da2dc7e86e7ad136567feb5e5eb5bf2dc0", "previousChainHash": "c43c1a7fbc4d242f13c91b1e41164943f9f509011e026ac027bc6886bd88614e" }, { "id": "4df4df82-fe5a-4a5b-b1b0-59553daffe48", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-09T02:17:16.108Z", "httpStatus": 301, "responseHeaders": { "content-type": "text/html; charset=UTF-8", "date": "Sun, 09 Aug 2026 02:17:15 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "4df4df82-fe5a-4a5b-b1b0-59553daffe48", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8eddc56cd2477c36b0e7a7487e4bf90842acea0fb361ee7f685a2fa9e98d6def", "previousChainHash": "29a2a4f96c42bdd248a7263b493ebefe51cd2c79b60f0350cfe8b73eedbfe915" }, { "id": "bc7a983a-0e69-444d-853e-6cd69dccc3cd", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-09T02:17:13.385Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Sun, 09 Aug 2026 02:17:12 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c43c1a7fbc4d242f13c91b1e41164943f9f509011e026ac027bc6886bd88614e", "previousChainHash": "522781aac5c99f4475d22f2d5ceb49594acc47db481c56dcc54af83a4e37edbe" }, { "id": "f174bd48-b2ae-46ed-bbd4-a8b9dff11c05", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-08T02:17:16.420Z", "httpStatus": 301, "responseHeaders": { "content-type": "text/html; charset=UTF-8", "date": "Sat, 08 Aug 2026 02:17:15 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "f174bd48-b2ae-46ed-bbd4-a8b9dff11c05", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "29a2a4f96c42bdd248a7263b493ebefe51cd2c79b60f0350cfe8b73eedbfe915", "previousChainHash": "a9155ce1627c726807d8217fef93d7893e7efbea80ee35d537895ccc45dbc181" }, { "id": "7e695d61-fe82-4cfc-8a7a-c43092d9706e", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-08T02:17:13.826Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Sat, 08 Aug 2026 02:17:12 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "522781aac5c99f4475d22f2d5ceb49594acc47db481c56dcc54af83a4e37edbe", "previousChainHash": "584b571b5ebbdea85f5eacace6ed1fd2ced5d5286f9cb2658efa5aa1a6e0a113" }, { "id": "54ecff57-c398-42c0-8334-4640175f7a50", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-07T02:17:16.642Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=3600", "content-length": "167", "content-type": "text/html", "date": "Fri, 07 Aug 2026 02:17:16 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "54ecff57-c398-42c0-8334-4640175f7a50", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a9155ce1627c726807d8217fef93d7893e7efbea80ee35d537895ccc45dbc181", "previousChainHash": "89e450a5c0a0097aa6dda6dc30f431ac5954a541a208e49e5fe962179fa6dd2e" }, { "id": "b7f4e4c3-481c-4379-8ce9-0cfc2bfca302", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-07T02:17:13.889Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Fri, 07 Aug 2026 02:17:13 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "584b571b5ebbdea85f5eacace6ed1fd2ced5d5286f9cb2658efa5aa1a6e0a113", "previousChainHash": "17a38aa3179c709ca25490fce39651419df651e5f44ff85ff3fad23698c5685d" }, { "id": "f5d3e41f-1b22-47d6-9279-546f07a6200c", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-06T02:17:16.934Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=3600", "content-length": "167", "content-type": "text/html", "date": "Thu, 06 Aug 2026 02:17:16 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "f5d3e41f-1b22-47d6-9279-546f07a6200c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "89e450a5c0a0097aa6dda6dc30f431ac5954a541a208e49e5fe962179fa6dd2e", "previousChainHash": "f81d8811550ca642d7bf6cac1c34432e6006d7d40c60ae329351a28278692812" }, { "id": "cbb30604-e895-4811-b326-4a889da0ae87", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-06T02:17:14.227Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Thu, 06 Aug 2026 02:17:13 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "17a38aa3179c709ca25490fce39651419df651e5f44ff85ff3fad23698c5685d", "previousChainHash": "e7ff7e37428b76c42fc9be6cb520fd5c38e5e8bdc8335cacf050d84ce827237f" }, { "id": "5970c552-2781-4d29-9e7b-54c93cc6bcbe", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-05T02:17:16.115Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=3600", "content-length": "167", "content-type": "text/html", "date": "Wed, 05 Aug 2026 02:17:16 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "5970c552-2781-4d29-9e7b-54c93cc6bcbe", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f81d8811550ca642d7bf6cac1c34432e6006d7d40c60ae329351a28278692812", "previousChainHash": "3c286825805248b7035bcd134ae60b9a28ad8a833fd8405d5813c3dbd30ded7e" }, { "id": "20d61ace-e9f5-4b13-b26e-78ddb56173ab", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-05T02:17:13.463Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Wed, 05 Aug 2026 02:17:13 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e7ff7e37428b76c42fc9be6cb520fd5c38e5e8bdc8335cacf050d84ce827237f", "previousChainHash": "896f11e2b3026b9568e0d2ca7fe6d7edea3a0adfaedbf89d6c1b25808095f11f" }, { "id": "b98b3d47-8558-4032-96cc-9ef14cb73a54", "domain": "cloudflare.com", "signal": "llms", "requestedUrl": "https://cloudflare.com/llms.txt", "finalUrl": "https://cloudflare.com/llms.txt", "fetchedAt": "2026-08-04T02:17:22.781Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=3600", "content-length": "167", "content-type": "text/html", "date": "Tue, 04 Aug 2026 02:17:22 GMT", "location": "https://www.cloudflare.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "b98b3d47-8558-4032-96cc-9ef14cb73a54", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3c286825805248b7035bcd134ae60b9a28ad8a833fd8405d5813c3dbd30ded7e", "previousChainHash": null }, { "id": "78c3b383-4307-494d-8357-c4d78a4a98c4", "domain": "cloudflare.com", "signal": "robots", "requestedUrl": "https://cloudflare.com/robots.txt", "finalUrl": "https://www.cloudflare.com/robots.txt", "fetchedAt": "2026-08-04T02:17:14.608Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "1099", "content-type": "text/plain", "date": "Tue, 04 Aug 2026 02:17:14 GMT", "etag": "\"0e9fa7a0bd53c344b463d30c8eb71478\"", "link": "; rel=\"api-catalog\", ; rel=\"service-desc\", ; rel=\"service-desc\", ; rel=\"service-doc\", ; rel=\"sitemap\"", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "artifactBytes": 1099, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ] }, { "agents": [ "Anthropic-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 24, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 27, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ] }, { "agents": [ "Cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 36, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.cloudflare.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "allow", "line": 30, "pattern": "/" } ], "state": "allowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "allow", "line": 18, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "allow", "line": 21, "pattern": "/" } ], "state": "allowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "allow", "line": 15, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 4, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "allow", "line": 33, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "78c3b383-4307-494d-8357-c4d78a4a98c4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "896f11e2b3026b9568e0d2ca7fe6d7edea3a0adfaedbf89d6c1b25808095f11f", "previousChainHash": null } ] }