{ "domain": "oreilly.com", "recordsNotConclusions": true, "observations": [ { "id": "0489747d-1b15-437d-81f5-6e8cdb5f030f", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-13T13:17:16.160Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Thu, 13 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "0489747d-1b15-437d-81f5-6e8cdb5f030f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "aa8717986029a3dc05fb99b59d6e91ede9eaa1d10d2799e063254aede619e6cd", "previousChainHash": "d5360e89254b27df677c4063fbd9027d349b8759ccfb652c872623dcd24d16c4" }, { "id": "3e506182-d732-481f-b4f4-a60ea17541ae", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-13T13:17:13.177Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=11171", "content-type": "text/plain; charset=utf-8", "date": "Thu, 13 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8935c3866c2207b00b8125398175e46c10adcd5f036c29c1b980e623ad5fc44f", "previousChainHash": "7164f96e0ea420fd8237ac93e4773e9636c8d60d009a47d45e15554e391c285a" } ], "payPerCrawl": { "status": "unverified", "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API." } }