{ "domain": "internetcollective.org", "recordsNotConclusions": true, "events": [ { "id": "0458e657-b5d0-402d-adbc-c73d953563b5", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-08T00:17:08.488Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Sun, 08 Nov 2026 00:17:08 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d6ab1f3e28742a53d73e6f7c791498a9d3755b7e3bada5f66d04aac51b02cc6a", "previousChainHash": "188cea91c39b66e0e0fae0b17efc0ba71a6a3c8dcf10f22f9684357143bb1c78" }, { "id": "81d07b67-88b9-44aa-896c-35c2cea61d8d", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-07T00:17:08.339Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Sat, 07 Nov 2026 00:17:08 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "188cea91c39b66e0e0fae0b17efc0ba71a6a3c8dcf10f22f9684357143bb1c78", "previousChainHash": "c845f40c667b6a75d1365414992f0a1e8ffc46dac1191f3b094337496debef71" }, { "id": "e7e1650b-8480-4b8a-8ef4-5f22223ae228", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-06T00:17:08.852Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Fri, 06 Nov 2026 00:17:08 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c845f40c667b6a75d1365414992f0a1e8ffc46dac1191f3b094337496debef71", "previousChainHash": "e6b849d485da384a730cc377099edc03e648135a9ddc5f81c77d68649dd9098b" }, { "id": "e098ee4a-a25d-477e-b38a-01fb8292780a", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-05T00:17:08.672Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Thu, 05 Nov 2026 00:17:08 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e6b849d485da384a730cc377099edc03e648135a9ddc5f81c77d68649dd9098b", "previousChainHash": "563b238bc307d6db7685eafaaa939d63e980cf3c296006964a66aedf6e30c688" }, { "id": "aeaa1a05-aca3-4e43-9ef2-7fcfa7598cda", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-04T00:17:07.752Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Wed, 04 Nov 2026 00:17:07 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "563b238bc307d6db7685eafaaa939d63e980cf3c296006964a66aedf6e30c688", "previousChainHash": "2d4e8f2b0b26469c6346d09502370769a834641d22fe64f2fffa5d1e861c4011" }, { "id": "e8020b5e-73cf-4dd1-bc94-d0b250cd6ba9", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-03T00:17:08.164Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Tue, 03 Nov 2026 00:17:08 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2d4e8f2b0b26469c6346d09502370769a834641d22fe64f2fffa5d1e861c4011", "previousChainHash": "cb0c2d9a03ef170e2b8d8bb1e9dd9defcae625d7f9976cd631360547b542add4" }, { "id": "753ef7c0-428c-4c2d-8816-8fda8b26848c", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-02T00:17:07.894Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Mon, 02 Nov 2026 00:17:07 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "cb0c2d9a03ef170e2b8d8bb1e9dd9defcae625d7f9976cd631360547b542add4", "previousChainHash": "a41b7d652b673ca6ca0235ad8bfc085d3895285f9b5f085a59f1841ed257e61b" }, { "id": "96947506-6482-4912-9b03-276bfda5893c", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-11-01T00:17:07.792Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Sun, 01 Nov 2026 00:17:07 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a41b7d652b673ca6ca0235ad8bfc085d3895285f9b5f085a59f1841ed257e61b", "previousChainHash": "b91c8147bd992f27be79933a04439a739dca598f6b3410cc3c28fce3b19705f7" }, { "id": "89ead7ac-0cd4-4d83-9190-093b0a657c37", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-10-31T00:17:09.062Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Sat, 31 Oct 2026 00:17:09 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "b91c8147bd992f27be79933a04439a739dca598f6b3410cc3c28fce3b19705f7", "previousChainHash": "bfe6efc184f333fd94a683f608c105fc514f05093d952b8c7ecb67b3056a450d" }, { "id": "f2774e30-b01c-413a-90cf-774bad608e3e", "domain": "internetcollective.org", "signal": "robots", "requestedUrl": "https://internetcollective.org/robots.txt", "finalUrl": "https://internetcollective.org/robots.txt", "fetchedAt": "2026-10-30T00:17:10.921Z", "httpStatus": 200, "responseHeaders": { "cache-control": "no-store", "content-length": "2676", "content-type": "text/plain; charset=utf-8", "date": "Fri, 30 Oct 2026 00:17:10 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add", "artifactBytes": 2676, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [ { "aiInput": null, "aiTrain": "no", "line": 31, "path": null, "search": "yes", "use": "reference" } ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Bytespider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "omgili" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] }, { "agents": [ "MistralAI-Training" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 72, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 75, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 78, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 81, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 84, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot", "AwarioRssBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "FishBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "BorderxBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" } ] }, { "agents": [ "NavuBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 112, "pattern": "/" } ] }, { "agents": [ "SemrushBot-SWA" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 115, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "KimiBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "CitibotSiteCrawler" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 32, "pattern": "/" } ], "state": "allowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bfe6efc184f333fd94a683f608c105fc514f05093d952b8c7ecb67b3056a450d", "previousChainHash": null } ] }