{
  "domain": "theguardian.com",
  "recordsNotConclusions": true,
  "observations": [
    {
      "id": "466fde42-1d2f-476a-bdeb-c5c05eff4f20",
      "domain": "theguardian.com",
      "signal": "llms",
      "requestedUrl": "https://theguardian.com/llms.txt",
      "finalUrl": "https://theguardian.com/llms.txt",
      "fetchedAt": "2026-08-13T16:17:22.376Z",
      "httpStatus": 301,
      "responseHeaders": {
        "content-length": "0",
        "date": "Thu, 13 Aug 2026 16:17:22 GMT",
        "location": "https://www.theguardian.com/llms.txt",
        "retry-after": "0",
        "server": "cloudflare"
      },
      "parseStatus": "http_error",
      "changed": false,
      "artifactSha256": null,
      "artifactBytes": null,
      "parserVersion": null,
      "parsed": null,
      "errorCode": "redirect_cross_host",
      "canonicalEventId": "466fde42-1d2f-476a-bdeb-c5c05eff4f20",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "3ec55c4c6f4b1b8cb5c3243fbd4c8b8c7be88ecf657ba86e9fbea179b166e52d",
      "previousChainHash": "3276ea8555e47eec93df1e565cbb261221a3326b7541adebf95cfee6f56ccb7c"
    },
    {
      "id": "aa2817bd-377c-4dea-b7d0-59e7c02736df",
      "domain": "theguardian.com",
      "signal": "robots",
      "requestedUrl": "https://theguardian.com/robots.txt",
      "finalUrl": "https://www.theguardian.com/robots.txt",
      "fetchedAt": "2026-08-13T16:17:14.036Z",
      "httpStatus": 200,
      "responseHeaders": {
        "cache-control": "max-age=300, private",
        "content-length": "2873",
        "content-type": "text/plain; charset=utf-8",
        "date": "Thu, 13 Aug 2026 16:17:14 GMT",
        "etag": "\"6a7c3e49-b39\"",
        "last-modified": "Wed, 12 Aug 2026 09:35:05 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "parsed",
      "changed": false,
      "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2",
      "artifactBytes": 2873,
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0",
      "parsed": {
        "globalLicenses": [],
        "groups": [
          {
            "agents": [
              "*"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/sendarticle/"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/Users/"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/users/"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/*/print$"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/email/"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/contactus/"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/share/"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/websearch"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/*?commentpage="
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/whsmiths/"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/external/overture/"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/discussion/report-abuse-ajax/*"
              },
              {
                "directive": "disallow",
                "line": 23,
                "pattern": "/discussion/comment-permalink/*"
              },
              {
                "directive": "disallow",
                "line": 24,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 25,
                "pattern": "/discussion/user-report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 26,
                "pattern": "/discussion/handlers/*"
              },
              {
                "directive": "disallow",
                "line": 27,
                "pattern": "/discussion/your-profile"
              },
              {
                "directive": "disallow",
                "line": 28,
                "pattern": "/discussion/your-comments"
              },
              {
                "directive": "disallow",
                "line": 29,
                "pattern": "/discussion/edit-profile"
              },
              {
                "directive": "disallow",
                "line": 30,
                "pattern": "/discussion/search/comments"
              },
              {
                "directive": "disallow",
                "line": 31,
                "pattern": "/discussion/*"
              },
              {
                "directive": "disallow",
                "line": 32,
                "pattern": "/search"
              },
              {
                "directive": "disallow",
                "line": 33,
                "pattern": "/music/artist/*"
              },
              {
                "directive": "disallow",
                "line": 34,
                "pattern": "/music/album/*"
              },
              {
                "directive": "disallow",
                "line": 35,
                "pattern": "/books/data/*"
              },
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/settings/"
              },
              {
                "directive": "disallow",
                "line": 37,
                "pattern": "/embed/"
              },
              {
                "directive": "disallow",
                "line": 38,
                "pattern": "/*styles/js-on.css$"
              },
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/sport/olympics/2008/events/*"
              },
              {
                "directive": "disallow",
                "line": 40,
                "pattern": "/sport/olympics/2008/medals/*"
              },
              {
                "directive": "disallow",
                "line": 41,
                "pattern": "/f/healthcheck"
              },
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/sections"
              },
              {
                "directive": "disallow",
                "line": 43,
                "pattern": "/top-stories"
              },
              {
                "directive": "disallow",
                "line": 44,
                "pattern": "/most-read/sport"
              },
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/articles"
              },
              {
                "directive": "disallow",
                "line": 46,
                "pattern": "/global$"
              },
              {
                "directive": "disallow",
                "line": 47,
                "pattern": "/*/feedarticle/*"
              },
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/travel/2013/aug/22/been-there-readers-competition?*"
              },
              {
                "directive": "disallow",
                "line": 49,
                "pattern": "/preference/*"
              },
              {
                "directive": "disallow",
                "line": 50,
                "pattern": "/59666047/"
              },
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/print/"
              },
              {
                "directive": "disallow",
                "line": 52,
                "pattern": "/info/tech-feedback"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/production-monitoring/"
              },
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "*.emailjson"
              },
              {
                "directive": "disallow",
                "line": 55,
                "pattern": "*.emailtxt"
              },
              {
                "directive": "disallow",
                "line": 56,
                "pattern": "/headline.txt"
              },
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "*?*dcr=apps*"
              }
            ]
          },
          {
            "agents": [
              "Mediapartners-Google"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": []
          },
          {
            "agents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "contentSignals": [],
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ]
          }
        ],
        "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0",
        "registryVersion": "2026-08-13",
        "sitemaps": [],
        "trackedAgents": {
          "Applebot-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "matchedAgents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "CCBot": {
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "matchedAgents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "ChatGPT-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/sendarticle/"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/Users/"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/users/"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/*/print$"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/email/"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/contactus/"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/share/"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/websearch"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/*?commentpage="
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/whsmiths/"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/external/overture/"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/discussion/report-abuse-ajax/*"
              },
              {
                "directive": "disallow",
                "line": 23,
                "pattern": "/discussion/comment-permalink/*"
              },
              {
                "directive": "disallow",
                "line": 24,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 25,
                "pattern": "/discussion/user-report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 26,
                "pattern": "/discussion/handlers/*"
              },
              {
                "directive": "disallow",
                "line": 27,
                "pattern": "/discussion/your-profile"
              },
              {
                "directive": "disallow",
                "line": 28,
                "pattern": "/discussion/your-comments"
              },
              {
                "directive": "disallow",
                "line": 29,
                "pattern": "/discussion/edit-profile"
              },
              {
                "directive": "disallow",
                "line": 30,
                "pattern": "/discussion/search/comments"
              },
              {
                "directive": "disallow",
                "line": 31,
                "pattern": "/discussion/*"
              },
              {
                "directive": "disallow",
                "line": 32,
                "pattern": "/search"
              },
              {
                "directive": "disallow",
                "line": 33,
                "pattern": "/music/artist/*"
              },
              {
                "directive": "disallow",
                "line": 34,
                "pattern": "/music/album/*"
              },
              {
                "directive": "disallow",
                "line": 35,
                "pattern": "/books/data/*"
              },
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/settings/"
              },
              {
                "directive": "disallow",
                "line": 37,
                "pattern": "/embed/"
              },
              {
                "directive": "disallow",
                "line": 38,
                "pattern": "/*styles/js-on.css$"
              },
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/sport/olympics/2008/events/*"
              },
              {
                "directive": "disallow",
                "line": 40,
                "pattern": "/sport/olympics/2008/medals/*"
              },
              {
                "directive": "disallow",
                "line": 41,
                "pattern": "/f/healthcheck"
              },
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/sections"
              },
              {
                "directive": "disallow",
                "line": 43,
                "pattern": "/top-stories"
              },
              {
                "directive": "disallow",
                "line": 44,
                "pattern": "/most-read/sport"
              },
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/articles"
              },
              {
                "directive": "disallow",
                "line": 46,
                "pattern": "/global$"
              },
              {
                "directive": "disallow",
                "line": 47,
                "pattern": "/*/feedarticle/*"
              },
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/travel/2013/aug/22/been-there-readers-competition?*"
              },
              {
                "directive": "disallow",
                "line": 49,
                "pattern": "/preference/*"
              },
              {
                "directive": "disallow",
                "line": 50,
                "pattern": "/59666047/"
              },
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/print/"
              },
              {
                "directive": "disallow",
                "line": 52,
                "pattern": "/info/tech-feedback"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/production-monitoring/"
              },
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "*.emailjson"
              },
              {
                "directive": "disallow",
                "line": 55,
                "pattern": "*.emailtxt"
              },
              {
                "directive": "disallow",
                "line": 56,
                "pattern": "/headline.txt"
              },
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "*?*dcr=apps*"
              }
            ],
            "state": "partial"
          },
          "Claude-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "matchedAgents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "Claude-User": {
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "matchedAgents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "ClaudeBot": {
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "matchedAgents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "Google-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/sendarticle/"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/Users/"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/users/"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/*/print$"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/email/"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/contactus/"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/share/"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/websearch"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/*?commentpage="
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/whsmiths/"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/external/overture/"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/discussion/report-abuse-ajax/*"
              },
              {
                "directive": "disallow",
                "line": 23,
                "pattern": "/discussion/comment-permalink/*"
              },
              {
                "directive": "disallow",
                "line": 24,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 25,
                "pattern": "/discussion/user-report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 26,
                "pattern": "/discussion/handlers/*"
              },
              {
                "directive": "disallow",
                "line": 27,
                "pattern": "/discussion/your-profile"
              },
              {
                "directive": "disallow",
                "line": 28,
                "pattern": "/discussion/your-comments"
              },
              {
                "directive": "disallow",
                "line": 29,
                "pattern": "/discussion/edit-profile"
              },
              {
                "directive": "disallow",
                "line": 30,
                "pattern": "/discussion/search/comments"
              },
              {
                "directive": "disallow",
                "line": 31,
                "pattern": "/discussion/*"
              },
              {
                "directive": "disallow",
                "line": 32,
                "pattern": "/search"
              },
              {
                "directive": "disallow",
                "line": 33,
                "pattern": "/music/artist/*"
              },
              {
                "directive": "disallow",
                "line": 34,
                "pattern": "/music/album/*"
              },
              {
                "directive": "disallow",
                "line": 35,
                "pattern": "/books/data/*"
              },
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/settings/"
              },
              {
                "directive": "disallow",
                "line": 37,
                "pattern": "/embed/"
              },
              {
                "directive": "disallow",
                "line": 38,
                "pattern": "/*styles/js-on.css$"
              },
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/sport/olympics/2008/events/*"
              },
              {
                "directive": "disallow",
                "line": 40,
                "pattern": "/sport/olympics/2008/medals/*"
              },
              {
                "directive": "disallow",
                "line": 41,
                "pattern": "/f/healthcheck"
              },
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/sections"
              },
              {
                "directive": "disallow",
                "line": 43,
                "pattern": "/top-stories"
              },
              {
                "directive": "disallow",
                "line": 44,
                "pattern": "/most-read/sport"
              },
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/articles"
              },
              {
                "directive": "disallow",
                "line": 46,
                "pattern": "/global$"
              },
              {
                "directive": "disallow",
                "line": 47,
                "pattern": "/*/feedarticle/*"
              },
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/travel/2013/aug/22/been-there-readers-competition?*"
              },
              {
                "directive": "disallow",
                "line": 49,
                "pattern": "/preference/*"
              },
              {
                "directive": "disallow",
                "line": 50,
                "pattern": "/59666047/"
              },
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/print/"
              },
              {
                "directive": "disallow",
                "line": 52,
                "pattern": "/info/tech-feedback"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/production-monitoring/"
              },
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "*.emailjson"
              },
              {
                "directive": "disallow",
                "line": 55,
                "pattern": "*.emailtxt"
              },
              {
                "directive": "disallow",
                "line": 56,
                "pattern": "/headline.txt"
              },
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "*?*dcr=apps*"
              }
            ],
            "state": "partial"
          },
          "GPTBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/sendarticle/"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/Users/"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/users/"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/*/print$"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/email/"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/contactus/"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/share/"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/websearch"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/*?commentpage="
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/whsmiths/"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/external/overture/"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/discussion/report-abuse-ajax/*"
              },
              {
                "directive": "disallow",
                "line": 23,
                "pattern": "/discussion/comment-permalink/*"
              },
              {
                "directive": "disallow",
                "line": 24,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 25,
                "pattern": "/discussion/user-report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 26,
                "pattern": "/discussion/handlers/*"
              },
              {
                "directive": "disallow",
                "line": 27,
                "pattern": "/discussion/your-profile"
              },
              {
                "directive": "disallow",
                "line": 28,
                "pattern": "/discussion/your-comments"
              },
              {
                "directive": "disallow",
                "line": 29,
                "pattern": "/discussion/edit-profile"
              },
              {
                "directive": "disallow",
                "line": 30,
                "pattern": "/discussion/search/comments"
              },
              {
                "directive": "disallow",
                "line": 31,
                "pattern": "/discussion/*"
              },
              {
                "directive": "disallow",
                "line": 32,
                "pattern": "/search"
              },
              {
                "directive": "disallow",
                "line": 33,
                "pattern": "/music/artist/*"
              },
              {
                "directive": "disallow",
                "line": 34,
                "pattern": "/music/album/*"
              },
              {
                "directive": "disallow",
                "line": 35,
                "pattern": "/books/data/*"
              },
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/settings/"
              },
              {
                "directive": "disallow",
                "line": 37,
                "pattern": "/embed/"
              },
              {
                "directive": "disallow",
                "line": 38,
                "pattern": "/*styles/js-on.css$"
              },
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/sport/olympics/2008/events/*"
              },
              {
                "directive": "disallow",
                "line": 40,
                "pattern": "/sport/olympics/2008/medals/*"
              },
              {
                "directive": "disallow",
                "line": 41,
                "pattern": "/f/healthcheck"
              },
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/sections"
              },
              {
                "directive": "disallow",
                "line": 43,
                "pattern": "/top-stories"
              },
              {
                "directive": "disallow",
                "line": 44,
                "pattern": "/most-read/sport"
              },
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/articles"
              },
              {
                "directive": "disallow",
                "line": 46,
                "pattern": "/global$"
              },
              {
                "directive": "disallow",
                "line": 47,
                "pattern": "/*/feedarticle/*"
              },
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/travel/2013/aug/22/been-there-readers-competition?*"
              },
              {
                "directive": "disallow",
                "line": 49,
                "pattern": "/preference/*"
              },
              {
                "directive": "disallow",
                "line": 50,
                "pattern": "/59666047/"
              },
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/print/"
              },
              {
                "directive": "disallow",
                "line": 52,
                "pattern": "/info/tech-feedback"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/production-monitoring/"
              },
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "*.emailjson"
              },
              {
                "directive": "disallow",
                "line": 55,
                "pattern": "*.emailtxt"
              },
              {
                "directive": "disallow",
                "line": 56,
                "pattern": "/headline.txt"
              },
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "*?*dcr=apps*"
              }
            ],
            "state": "partial"
          },
          "OAI-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/sendarticle/"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/Users/"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/users/"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/*/print$"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/email/"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/contactus/"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/share/"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/websearch"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/*?commentpage="
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/whsmiths/"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/external/overture/"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/discussion/report-abuse-ajax/*"
              },
              {
                "directive": "disallow",
                "line": 23,
                "pattern": "/discussion/comment-permalink/*"
              },
              {
                "directive": "disallow",
                "line": 24,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 25,
                "pattern": "/discussion/user-report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 26,
                "pattern": "/discussion/handlers/*"
              },
              {
                "directive": "disallow",
                "line": 27,
                "pattern": "/discussion/your-profile"
              },
              {
                "directive": "disallow",
                "line": 28,
                "pattern": "/discussion/your-comments"
              },
              {
                "directive": "disallow",
                "line": 29,
                "pattern": "/discussion/edit-profile"
              },
              {
                "directive": "disallow",
                "line": 30,
                "pattern": "/discussion/search/comments"
              },
              {
                "directive": "disallow",
                "line": 31,
                "pattern": "/discussion/*"
              },
              {
                "directive": "disallow",
                "line": 32,
                "pattern": "/search"
              },
              {
                "directive": "disallow",
                "line": 33,
                "pattern": "/music/artist/*"
              },
              {
                "directive": "disallow",
                "line": 34,
                "pattern": "/music/album/*"
              },
              {
                "directive": "disallow",
                "line": 35,
                "pattern": "/books/data/*"
              },
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/settings/"
              },
              {
                "directive": "disallow",
                "line": 37,
                "pattern": "/embed/"
              },
              {
                "directive": "disallow",
                "line": 38,
                "pattern": "/*styles/js-on.css$"
              },
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/sport/olympics/2008/events/*"
              },
              {
                "directive": "disallow",
                "line": 40,
                "pattern": "/sport/olympics/2008/medals/*"
              },
              {
                "directive": "disallow",
                "line": 41,
                "pattern": "/f/healthcheck"
              },
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/sections"
              },
              {
                "directive": "disallow",
                "line": 43,
                "pattern": "/top-stories"
              },
              {
                "directive": "disallow",
                "line": 44,
                "pattern": "/most-read/sport"
              },
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/articles"
              },
              {
                "directive": "disallow",
                "line": 46,
                "pattern": "/global$"
              },
              {
                "directive": "disallow",
                "line": 47,
                "pattern": "/*/feedarticle/*"
              },
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/travel/2013/aug/22/been-there-readers-competition?*"
              },
              {
                "directive": "disallow",
                "line": 49,
                "pattern": "/preference/*"
              },
              {
                "directive": "disallow",
                "line": 50,
                "pattern": "/59666047/"
              },
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/print/"
              },
              {
                "directive": "disallow",
                "line": 52,
                "pattern": "/info/tech-feedback"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/production-monitoring/"
              },
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "*.emailjson"
              },
              {
                "directive": "disallow",
                "line": 55,
                "pattern": "*.emailtxt"
              },
              {
                "directive": "disallow",
                "line": 56,
                "pattern": "/headline.txt"
              },
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "*?*dcr=apps*"
              }
            ],
            "state": "partial"
          },
          "Perplexity-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/sendarticle/"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/Users/"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/users/"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/*/print$"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/email/"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/contactus/"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/share/"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/websearch"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/*?commentpage="
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/whsmiths/"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/external/overture/"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/discussion/report-abuse-ajax/*"
              },
              {
                "directive": "disallow",
                "line": 23,
                "pattern": "/discussion/comment-permalink/*"
              },
              {
                "directive": "disallow",
                "line": 24,
                "pattern": "/discussion/report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 25,
                "pattern": "/discussion/user-report-abuse/*"
              },
              {
                "directive": "disallow",
                "line": 26,
                "pattern": "/discussion/handlers/*"
              },
              {
                "directive": "disallow",
                "line": 27,
                "pattern": "/discussion/your-profile"
              },
              {
                "directive": "disallow",
                "line": 28,
                "pattern": "/discussion/your-comments"
              },
              {
                "directive": "disallow",
                "line": 29,
                "pattern": "/discussion/edit-profile"
              },
              {
                "directive": "disallow",
                "line": 30,
                "pattern": "/discussion/search/comments"
              },
              {
                "directive": "disallow",
                "line": 31,
                "pattern": "/discussion/*"
              },
              {
                "directive": "disallow",
                "line": 32,
                "pattern": "/search"
              },
              {
                "directive": "disallow",
                "line": 33,
                "pattern": "/music/artist/*"
              },
              {
                "directive": "disallow",
                "line": 34,
                "pattern": "/music/album/*"
              },
              {
                "directive": "disallow",
                "line": 35,
                "pattern": "/books/data/*"
              },
              {
                "directive": "disallow",
                "line": 36,
                "pattern": "/settings/"
              },
              {
                "directive": "disallow",
                "line": 37,
                "pattern": "/embed/"
              },
              {
                "directive": "disallow",
                "line": 38,
                "pattern": "/*styles/js-on.css$"
              },
              {
                "directive": "disallow",
                "line": 39,
                "pattern": "/sport/olympics/2008/events/*"
              },
              {
                "directive": "disallow",
                "line": 40,
                "pattern": "/sport/olympics/2008/medals/*"
              },
              {
                "directive": "disallow",
                "line": 41,
                "pattern": "/f/healthcheck"
              },
              {
                "directive": "disallow",
                "line": 42,
                "pattern": "/sections"
              },
              {
                "directive": "disallow",
                "line": 43,
                "pattern": "/top-stories"
              },
              {
                "directive": "disallow",
                "line": 44,
                "pattern": "/most-read/sport"
              },
              {
                "directive": "disallow",
                "line": 45,
                "pattern": "/articles"
              },
              {
                "directive": "disallow",
                "line": 46,
                "pattern": "/global$"
              },
              {
                "directive": "disallow",
                "line": 47,
                "pattern": "/*/feedarticle/*"
              },
              {
                "directive": "disallow",
                "line": 48,
                "pattern": "/travel/2013/aug/22/been-there-readers-competition?*"
              },
              {
                "directive": "disallow",
                "line": 49,
                "pattern": "/preference/*"
              },
              {
                "directive": "disallow",
                "line": 50,
                "pattern": "/59666047/"
              },
              {
                "directive": "disallow",
                "line": 51,
                "pattern": "/print/"
              },
              {
                "directive": "disallow",
                "line": 52,
                "pattern": "/info/tech-feedback"
              },
              {
                "directive": "disallow",
                "line": 53,
                "pattern": "/production-monitoring/"
              },
              {
                "directive": "disallow",
                "line": 54,
                "pattern": "*.emailjson"
              },
              {
                "directive": "disallow",
                "line": 55,
                "pattern": "*.emailtxt"
              },
              {
                "directive": "disallow",
                "line": 56,
                "pattern": "/headline.txt"
              },
              {
                "directive": "disallow",
                "line": 57,
                "pattern": "*?*dcr=apps*"
              }
            ],
            "state": "partial"
          },
          "PerplexityBot": {
            "crawlDelaySeconds": null,
            "licenses": [
              "https://theguardian.com/license.xml"
            ],
            "matchedAgents": [
              "NewsNow",
              "CCBot",
              "TurnitinBot",
              "PetalBot",
              "MoodleBot",
              "FacebookBot",
              "Bytespider",
              "Mojeek",
              "JenkersBot",
              "Seekr",
              "YouBot",
              "Arquivo-web-crawler",
              "coccocbot-web",
              "SeznamBot",
              "PerplexityBot",
              "yacy",
              "anthropic-ai",
              "ClaudeBot",
              "Claude-SearchBot",
              "Claude-User",
              "AwarioRssBot",
              "AwarioSmartBot",
              "SentiOne",
              "ImageSift",
              "Applebot-Extended",
              "YandexAdditional",
              "YandexAdditionalBot",
              "scalepostAI",
              "Buck",
              "meta-externalagent",
              "Amazonbot",
              "amazon-QBusiness",
              "DuckAssistBot",
              "Google-CloudVertexBot",
              "Amzn-SearchBot",
              "AhrefsBot",
              "AhrefsSiteAudit"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 102,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          }
        },
        "warnings": [
          "line 62: non-HTTPS or invalid sitemap ignored",
          "line 63: non-HTTPS or invalid sitemap ignored"
        ]
      },
      "errorCode": null,
      "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "cbce7f07aacc5296fa247dc392b49b25eb680b2afc29a16590c2e192688d776a",
      "previousChainHash": "f9d701d74f22123e3b81395fe58d0aadd1cc4ebf6b319d8bd7eac909577be1d7"
    }
  ],
  "payPerCrawl": {
    "status": "unverified",
    "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API."
  }
}