{
  "domain": "internetcollective.org",
  "recordsNotConclusions": true,
  "observations": [
    {
      "id": "0458e657-b5d0-402d-adbc-c73d953563b5",
      "domain": "internetcollective.org",
      "signal": "robots",
      "requestedUrl": "https://internetcollective.org/robots.txt",
      "finalUrl": "https://internetcollective.org/robots.txt",
      "fetchedAt": "2026-11-08T00:17:08.488Z",
      "httpStatus": 200,
      "responseHeaders": {
        "cache-control": "no-store",
        "content-length": "2676",
        "content-type": "text/plain; charset=utf-8",
        "date": "Sun, 08 Nov 2026 00:17:08 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "parsed",
      "changed": false,
      "artifactSha256": "3073d1e371afb3271c7666148766fedc5145f7d8440f7114062db79b7bd58add",
      "artifactBytes": 2676,
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20",
      "parsed": {
        "globalLicenses": [],
        "groups": [
          {
            "agents": [
              "*"
            ],
            "contentSignals": [
              {
                "aiInput": null,
                "aiTrain": "no",
                "line": 31,
                "path": null,
                "search": "yes",
                "use": "reference"
              }
            ],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Amazonbot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Applebot-Extended"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Bytespider"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "CCBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "ClaudeBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Diffbot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Google-Extended"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "GPTBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "omgili"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 60,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "anthropic-ai"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 63,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Claude-Web"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 66,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "cohere-ai"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 69,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "MistralAI-Training"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 72,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "meta-externalagent"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 75,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "GoogleOther"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 78,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Baiduspider"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 81,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "PetalBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 84,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "AwarioSmartBot",
              "AwarioRssBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 88,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Google-CloudVertexBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 91,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "QualifiedBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 94,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Cotoyogi"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 97,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "ICC-Crawler"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 100,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "atlassian-bot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 103,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "FishBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 106,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "BorderxBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 109,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "NavuBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 112,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "SemrushBot-SWA"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 115,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "WARDBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 118,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "magpie-crawler"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 121,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "KimiBot"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 124,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "CitibotSiteCrawler"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 127,
                "pattern": "/"
              }
            ]
          }
        ],
        "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20",
        "registryVersion": "2026-09-20",
        "sitemaps": [],
        "trackedAgents": {
          "Applebot-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "Applebot-Extended"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "CCBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "CCBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "ChatGPT-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          },
          "Claude-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          },
          "Claude-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          },
          "ClaudeBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ClaudeBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "Google-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "Google-Extended"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "GPTBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "GPTBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "OAI-AdsBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          },
          "OAI-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          },
          "Perplexity-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          },
          "PerplexityBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "allow",
                "line": 32,
                "pattern": "/"
              }
            ],
            "state": "allowed"
          }
        },
        "warnings": []
      },
      "errorCode": null,
      "canonicalEventId": "f2774e30-b01c-413a-90cf-774bad608e3e",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "d6ab1f3e28742a53d73e6f7c791498a9d3755b7e3bada5f66d04aac51b02cc6a",
      "previousChainHash": "188cea91c39b66e0e0fae0b17efc0ba71a6a3c8dcf10f22f9684357143bb1c78"
    }
  ],
  "payPerCrawl": {
    "status": "unverified",
    "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API."
  }
}