{ "domain": "theguardian.com", "recordsNotConclusions": true, "events": [ { "id": "466fde42-1d2f-476a-bdeb-c5c05eff4f20", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-13T16:17:22.376Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Thu, 13 Aug 2026 16:17:22 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "466fde42-1d2f-476a-bdeb-c5c05eff4f20", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3ec55c4c6f4b1b8cb5c3243fbd4c8b8c7be88ecf657ba86e9fbea179b166e52d", "previousChainHash": "3276ea8555e47eec93df1e565cbb261221a3326b7541adebf95cfee6f56ccb7c" }, { "id": "aa2817bd-377c-4dea-b7d0-59e7c02736df", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-13T16:17:14.036Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Thu, 13 Aug 2026 16:17:14 GMT", "etag": "\"6a7c3e49-b39\"", "last-modified": "Wed, 12 Aug 2026 09:35:05 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0", "registryVersion": "2026-08-13", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "cbce7f07aacc5296fa247dc392b49b25eb680b2afc29a16590c2e192688d776a", "previousChainHash": "f9d701d74f22123e3b81395fe58d0aadd1cc4ebf6b319d8bd7eac909577be1d7" }, { "id": "d9434eaf-2abe-4807-a5c6-1311cbd99f48", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-12T16:17:17.403Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Wed, 12 Aug 2026 16:17:17 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "d9434eaf-2abe-4807-a5c6-1311cbd99f48", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3276ea8555e47eec93df1e565cbb261221a3326b7541adebf95cfee6f56ccb7c", "previousChainHash": "22f8ffec5731e4024bcf1a15cceef9c4d8f6416c800caf3ae4517fdf52ea039b" }, { "id": "0f7c0230-c227-4ff9-9663-a993cf9c4595", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-12T16:17:14.625Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Wed, 12 Aug 2026 16:17:14 GMT", "etag": "\"6a7c3e49-b39\"", "last-modified": "Wed, 12 Aug 2026 09:35:05 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f9d701d74f22123e3b81395fe58d0aadd1cc4ebf6b319d8bd7eac909577be1d7", "previousChainHash": "81212f841a85c287ec75d4f37e495bf8112a131aa823f79ae1d36e21765f350f" }, { "id": "b94b6240-d410-476a-9acb-a427c25ffd4b", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-11T16:17:16.500Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Tue, 11 Aug 2026 16:17:16 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "b94b6240-d410-476a-9acb-a427c25ffd4b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "22f8ffec5731e4024bcf1a15cceef9c4d8f6416c800caf3ae4517fdf52ea039b", "previousChainHash": "6df1a6519cb68293feda6f26293b8e667f2a13879e81fff46caafb20d63cf22b" }, { "id": "37d78f63-af4b-4024-982b-d052ecccc183", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-11T16:17:13.713Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Tue, 11 Aug 2026 16:17:13 GMT", "etag": "\"6a799b61-b39\"", "last-modified": "Mon, 10 Aug 2026 09:35:29 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "81212f841a85c287ec75d4f37e495bf8112a131aa823f79ae1d36e21765f350f", "previousChainHash": "58193497f146bc4cbc671a77f571046d5a573e6e2e07ec585c801751432e4b65" }, { "id": "239cfe93-ca8b-404e-af12-9b0408172337", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-10T16:17:22.577Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Mon, 10 Aug 2026 16:17:22 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "239cfe93-ca8b-404e-af12-9b0408172337", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6df1a6519cb68293feda6f26293b8e667f2a13879e81fff46caafb20d63cf22b", "previousChainHash": "07a707fc8daa990cfaa942ca60e0552aaf0ab288b7d285e2fbd771236d8f551b" }, { "id": "e78e81d3-2d22-4be8-b681-9fb8ca0b13d2", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-10T16:17:14.758Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Mon, 10 Aug 2026 16:17:14 GMT", "etag": "\"6a799b61-b39\"", "last-modified": "Mon, 10 Aug 2026 09:35:29 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "58193497f146bc4cbc671a77f571046d5a573e6e2e07ec585c801751432e4b65", "previousChainHash": "436c77182995035217b194e7765fafc6fb2b6b913306f100ba33522c55ba481a" }, { "id": "faeb6c58-8ab4-4f17-a993-ef9a6581f3a2", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-09T16:17:16.665Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Sun, 09 Aug 2026 16:17:16 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "faeb6c58-8ab4-4f17-a993-ef9a6581f3a2", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "07a707fc8daa990cfaa942ca60e0552aaf0ab288b7d285e2fbd771236d8f551b", "previousChainHash": "5d7a4b3736cb3268742772d276bbc96d2c5997f041b2d99805841d8bbe0f36ae" }, { "id": "84728719-25ad-4f33-8ee1-e55e5618117b", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-09T16:17:13.859Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Sun, 09 Aug 2026 16:17:13 GMT", "etag": "\"6a75a6c9-b39\"", "last-modified": "Fri, 07 Aug 2026 09:35:05 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "436c77182995035217b194e7765fafc6fb2b6b913306f100ba33522c55ba481a", "previousChainHash": "81426569024712f806fe132be2ea78f54ae78594bb95a82aebf8e8902b4a6c0b" }, { "id": "5ff0d651-39ee-417a-8fc1-c4a338dbe44a", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-08T16:17:16.324Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Sat, 08 Aug 2026 16:17:16 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "5ff0d651-39ee-417a-8fc1-c4a338dbe44a", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "5d7a4b3736cb3268742772d276bbc96d2c5997f041b2d99805841d8bbe0f36ae", "previousChainHash": "c0fe5246572db0f690c43d9aa12885a59b6392e7ee30d192035d2640578c43b7" }, { "id": "4fdfd384-a62d-402f-b346-d59298b90b7c", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-08T16:17:13.679Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Sat, 08 Aug 2026 16:17:13 GMT", "etag": "\"6a75a6c9-b39\"", "last-modified": "Fri, 07 Aug 2026 09:35:05 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "81426569024712f806fe132be2ea78f54ae78594bb95a82aebf8e8902b4a6c0b", "previousChainHash": "f8483bed95298a579500d5b5f60518bf8c1ab54e8972f3c3632ada5926a9ceef" }, { "id": "84269e57-f082-49c8-94ff-d76c5ade27df", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-07T16:17:16.104Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Fri, 07 Aug 2026 16:17:16 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "84269e57-f082-49c8-94ff-d76c5ade27df", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c0fe5246572db0f690c43d9aa12885a59b6392e7ee30d192035d2640578c43b7", "previousChainHash": "2503170e7107d278b890beed7fcd8847d05e31408893f0037ebc25e07d591d30" }, { "id": "4b2ece31-ebcb-49af-a2fd-13a3e073afe9", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-07T16:17:13.384Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Fri, 07 Aug 2026 16:17:13 GMT", "etag": "\"6a75a6c9-b39\"", "last-modified": "Fri, 07 Aug 2026 09:35:05 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f8483bed95298a579500d5b5f60518bf8c1ab54e8972f3c3632ada5926a9ceef", "previousChainHash": "452754524d995b387366048d6645fcf3328135d1220178e5f8eb8d2dbe1c615c" }, { "id": "099f88db-b766-4bec-97a9-e6c046447666", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-06T16:17:16.516Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Thu, 06 Aug 2026 16:17:16 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "099f88db-b766-4bec-97a9-e6c046447666", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2503170e7107d278b890beed7fcd8847d05e31408893f0037ebc25e07d591d30", "previousChainHash": "1d380f3e4d6b09e338f4462c8b31b2a36149e40b5569febc2b80faf17e2ffa33" }, { "id": "84155433-71be-44d9-87fb-8e0909087f96", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-06T16:17:13.798Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Thu, 06 Aug 2026 16:17:13 GMT", "etag": "\"6a7303da-b39\"", "last-modified": "Wed, 05 Aug 2026 09:35:22 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "452754524d995b387366048d6645fcf3328135d1220178e5f8eb8d2dbe1c615c", "previousChainHash": "f507c9a31c7d7857ab753b1c34e68d63df88d860472001a2da014d003b1c71e5" }, { "id": "d00ff82b-7bba-45de-9679-ca04a0bc61dc", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-05T16:17:16.439Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Wed, 05 Aug 2026 16:17:16 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "d00ff82b-7bba-45de-9679-ca04a0bc61dc", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1d380f3e4d6b09e338f4462c8b31b2a36149e40b5569febc2b80faf17e2ffa33", "previousChainHash": "088aebae49de8757f4ef7d85e1e332008b223f4a08d69f3628caa68010123987" }, { "id": "edb76d3f-a34f-470f-9314-1de66625669b", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-05T16:17:13.737Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Wed, 05 Aug 2026 16:17:13 GMT", "etag": "\"6a7303da-b39\"", "last-modified": "Wed, 05 Aug 2026 09:35:22 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f507c9a31c7d7857ab753b1c34e68d63df88d860472001a2da014d003b1c71e5", "previousChainHash": "d8bdb8e0057a60c096d645e9385bd18eac651307443ec611b244e8e14c850e62" }, { "id": "71ac2e93-3d57-42e3-b534-db5b4d4ef58a", "domain": "theguardian.com", "signal": "llms", "requestedUrl": "https://theguardian.com/llms.txt", "finalUrl": "https://theguardian.com/llms.txt", "fetchedAt": "2026-08-04T16:17:22.822Z", "httpStatus": 301, "responseHeaders": { "content-length": "0", "date": "Tue, 04 Aug 2026 16:17:22 GMT", "location": "https://www.theguardian.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "71ac2e93-3d57-42e3-b534-db5b4d4ef58a", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "088aebae49de8757f4ef7d85e1e332008b223f4a08d69f3628caa68010123987", "previousChainHash": null }, { "id": "4244cb76-4150-44d4-973b-a57dc5f9c344", "domain": "theguardian.com", "signal": "robots", "requestedUrl": "https://theguardian.com/robots.txt", "finalUrl": "https://www.theguardian.com/robots.txt", "fetchedAt": "2026-08-04T16:17:14.445Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=300, private", "content-length": "2873", "content-type": "text/plain; charset=utf-8", "date": "Tue, 04 Aug 2026 16:17:14 GMT", "etag": "\"6a7060e7-b39\"", "last-modified": "Mon, 03 Aug 2026 09:35:35 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "artifactBytes": 2873, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ] }, { "agents": [ "Mediapartners-Google" ], "crawlDelaySeconds": null, "licenses": [], "rules": [] }, { "agents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/sendarticle/" }, { "directive": "disallow", "line": 11, "pattern": "/Users/" }, { "directive": "disallow", "line": 12, "pattern": "/users/" }, { "directive": "disallow", "line": 13, "pattern": "/*/print$" }, { "directive": "disallow", "line": 14, "pattern": "/email/" }, { "directive": "disallow", "line": 15, "pattern": "/contactus/" }, { "directive": "disallow", "line": 16, "pattern": "/share/" }, { "directive": "disallow", "line": 17, "pattern": "/websearch" }, { "directive": "disallow", "line": 18, "pattern": "/*?commentpage=" }, { "directive": "disallow", "line": 19, "pattern": "/whsmiths/" }, { "directive": "disallow", "line": 20, "pattern": "/external/overture/" }, { "directive": "disallow", "line": 21, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 22, "pattern": "/discussion/report-abuse-ajax/*" }, { "directive": "disallow", "line": 23, "pattern": "/discussion/comment-permalink/*" }, { "directive": "disallow", "line": 24, "pattern": "/discussion/report-abuse/*" }, { "directive": "disallow", "line": 25, "pattern": "/discussion/user-report-abuse/*" }, { "directive": "disallow", "line": 26, "pattern": "/discussion/handlers/*" }, { "directive": "disallow", "line": 27, "pattern": "/discussion/your-profile" }, { "directive": "disallow", "line": 28, "pattern": "/discussion/your-comments" }, { "directive": "disallow", "line": 29, "pattern": "/discussion/edit-profile" }, { "directive": "disallow", "line": 30, "pattern": "/discussion/search/comments" }, { "directive": "disallow", "line": 31, "pattern": "/discussion/*" }, { "directive": "disallow", "line": 32, "pattern": "/search" }, { "directive": "disallow", "line": 33, "pattern": "/music/artist/*" }, { "directive": "disallow", "line": 34, "pattern": "/music/album/*" }, { "directive": "disallow", "line": 35, "pattern": "/books/data/*" }, { "directive": "disallow", "line": 36, "pattern": "/settings/" }, { "directive": "disallow", "line": 37, "pattern": "/embed/" }, { "directive": "disallow", "line": 38, "pattern": "/*styles/js-on.css$" }, { "directive": "disallow", "line": 39, "pattern": "/sport/olympics/2008/events/*" }, { "directive": "disallow", "line": 40, "pattern": "/sport/olympics/2008/medals/*" }, { "directive": "disallow", "line": 41, "pattern": "/f/healthcheck" }, { "directive": "disallow", "line": 42, "pattern": "/sections" }, { "directive": "disallow", "line": 43, "pattern": "/top-stories" }, { "directive": "disallow", "line": 44, "pattern": "/most-read/sport" }, { "directive": "disallow", "line": 45, "pattern": "/articles" }, { "directive": "disallow", "line": 46, "pattern": "/global$" }, { "directive": "disallow", "line": 47, "pattern": "/*/feedarticle/*" }, { "directive": "disallow", "line": 48, "pattern": "/travel/2013/aug/22/been-there-readers-competition?*" }, { "directive": "disallow", "line": 49, "pattern": "/preference/*" }, { "directive": "disallow", "line": 50, "pattern": "/59666047/" }, { "directive": "disallow", "line": 51, "pattern": "/print/" }, { "directive": "disallow", "line": 52, "pattern": "/info/tech-feedback" }, { "directive": "disallow", "line": 53, "pattern": "/production-monitoring/" }, { "directive": "disallow", "line": 54, "pattern": "*.emailjson" }, { "directive": "disallow", "line": 55, "pattern": "*.emailtxt" }, { "directive": "disallow", "line": 56, "pattern": "/headline.txt" }, { "directive": "disallow", "line": 57, "pattern": "*?*dcr=apps*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [ "https://theguardian.com/license.xml" ], "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "rules": [ { "directive": "disallow", "line": 102, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [ "line 62: non-HTTPS or invalid sitemap ignored", "line 63: non-HTTPS or invalid sitemap ignored" ] }, "errorCode": null, "canonicalEventId": "4244cb76-4150-44d4-973b-a57dc5f9c344", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d8bdb8e0057a60c096d645e9385bd18eac651307443ec611b244e8e14c850e62", "previousChainHash": null } ] }