{ "domain": "oreilly.com", "recordsNotConclusions": true, "events": [ { "id": "0489747d-1b15-437d-81f5-6e8cdb5f030f", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-13T13:17:16.160Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Thu, 13 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "0489747d-1b15-437d-81f5-6e8cdb5f030f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "aa8717986029a3dc05fb99b59d6e91ede9eaa1d10d2799e063254aede619e6cd", "previousChainHash": "d5360e89254b27df677c4063fbd9027d349b8759ccfb652c872623dcd24d16c4" }, { "id": "3e506182-d732-481f-b4f4-a60ea17541ae", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-13T13:17:13.177Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=11171", "content-type": "text/plain; charset=utf-8", "date": "Thu, 13 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8935c3866c2207b00b8125398175e46c10adcd5f036c29c1b980e623ad5fc44f", "previousChainHash": "7164f96e0ea420fd8237ac93e4773e9636c8d60d009a47d45e15554e391c285a" }, { "id": "b375f429-5e9f-4699-81b7-6fa85e86512a", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-12T13:17:16.151Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Wed, 12 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "b375f429-5e9f-4699-81b7-6fa85e86512a", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d5360e89254b27df677c4063fbd9027d349b8759ccfb652c872623dcd24d16c4", "previousChainHash": "51eb2419e743f0e01a3f2d40c104b1a6d5a2c316f8cf2b3e0d7fab6a3d4289ed" }, { "id": "afb3555c-6e89-430e-8832-5521711888d9", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-12T13:17:13.405Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=9973", "content-type": "text/plain; charset=utf-8", "date": "Wed, 12 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7164f96e0ea420fd8237ac93e4773e9636c8d60d009a47d45e15554e391c285a", "previousChainHash": "00a93972cf10fe99e05904dfcf875b79b18cc72f479cb2ca4e33377a6eda5348" }, { "id": "baace97a-21f0-4721-9c7e-a77eb43e2858", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-11T13:17:16.188Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Tue, 11 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "baace97a-21f0-4721-9c7e-a77eb43e2858", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "51eb2419e743f0e01a3f2d40c104b1a6d5a2c316f8cf2b3e0d7fab6a3d4289ed", "previousChainHash": "042ad2660a9960f3e9a37e5c94f85477efdb0176c023cb946a842f5e6ee4f2f3" }, { "id": "d28e5f47-4110-498c-a640-1bd0d4e363d6", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-11T13:17:13.447Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=637", "content-type": "text/plain; charset=utf-8", "date": "Tue, 11 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "00a93972cf10fe99e05904dfcf875b79b18cc72f479cb2ca4e33377a6eda5348", "previousChainHash": "c56a48c1d31bf56d5889fb7c9159b6da69a06802997d46580d006fa44551800b" }, { "id": "f5eac386-095e-4ea1-8d13-cfd7f4e609ab", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-10T13:17:15.648Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Mon, 10 Aug 2026 13:17:15 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "f5eac386-095e-4ea1-8d13-cfd7f4e609ab", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "042ad2660a9960f3e9a37e5c94f85477efdb0176c023cb946a842f5e6ee4f2f3", "previousChainHash": "d4cac75ee92f088d1c78682f1779b03e57b82852f5551cf2c439ac12ca23e4f1" }, { "id": "5271f43f-df7e-4a04-904b-73801044df72", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-10T13:17:12.925Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=4511", "content-type": "text/plain; charset=utf-8", "date": "Mon, 10 Aug 2026 13:17:12 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c56a48c1d31bf56d5889fb7c9159b6da69a06802997d46580d006fa44551800b", "previousChainHash": "3ff1c73d14dd58ff1bbf829d64c244a83fd3db359e5713dbde2c711168dca0af" }, { "id": "6e5abd20-6835-4f28-b454-b4bb8d79ebf4", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-09T13:17:16.276Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Sun, 09 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "6e5abd20-6835-4f28-b454-b4bb8d79ebf4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d4cac75ee92f088d1c78682f1779b03e57b82852f5551cf2c439ac12ca23e4f1", "previousChainHash": "39b6b4c61dfa53489bef0ddb69d621be30f8c3375bf5f46d010f57ddae456e0f" }, { "id": "4a78e11c-2470-4823-a6b7-dda9493482fc", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-09T13:17:13.483Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=8980", "content-type": "text/plain; charset=utf-8", "date": "Sun, 09 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3ff1c73d14dd58ff1bbf829d64c244a83fd3db359e5713dbde2c711168dca0af", "previousChainHash": "505ed144e0d162a37a57161f2ff62f8e90d69304cb048f0a36e91a6840df51f0" }, { "id": "0c0338fc-a3a9-4071-82d8-fcdf85edec62", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-08T13:17:16.614Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Sat, 08 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "0c0338fc-a3a9-4071-82d8-fcdf85edec62", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "39b6b4c61dfa53489bef0ddb69d621be30f8c3375bf5f46d010f57ddae456e0f", "previousChainHash": "ad555c873a3dd5bc0a7af4183879724bbe171e7ca1560715862d64642b3f93eb" }, { "id": "e7f6498b-a02c-46c9-889c-1531112253d3", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-08T13:17:13.837Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=14057", "content-type": "text/plain; charset=utf-8", "date": "Sat, 08 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "505ed144e0d162a37a57161f2ff62f8e90d69304cb048f0a36e91a6840df51f0", "previousChainHash": "caf4491b6746b222c82c696e7b7e9bb13d1ce6b84e57bb71c12cd3c24dded7c3" }, { "id": "f7fcbd62-d329-4fc9-9e9e-b01835bd652b", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-07T13:17:16.885Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Fri, 07 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "f7fcbd62-d329-4fc9-9e9e-b01835bd652b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ad555c873a3dd5bc0a7af4183879724bbe171e7ca1560715862d64642b3f93eb", "previousChainHash": "286985ed121067a09534111a3668d18664afe336d5c46cb8bf910e5735b76205" }, { "id": "43a2f9eb-ac7f-4682-8912-a58a59fbfd5a", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-07T13:17:14.193Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=6487", "content-type": "text/plain; charset=utf-8", "date": "Fri, 07 Aug 2026 13:17:14 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "caf4491b6746b222c82c696e7b7e9bb13d1ce6b84e57bb71c12cd3c24dded7c3", "previousChainHash": "c6650b8d7911755d0db0566b10abd41ae0147b15bc900c9a47e13a77e9b829d3" }, { "id": "8ac52da9-102f-4c0d-bdbb-44fef156e83e", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-06T13:17:17.046Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Thu, 06 Aug 2026 13:17:17 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "8ac52da9-102f-4c0d-bdbb-44fef156e83e", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "286985ed121067a09534111a3668d18664afe336d5c46cb8bf910e5735b76205", "previousChainHash": "902b323b0abe1661a07f32bcdef098f3a7a601efda9cd2628f5c8f6cdeeda64e" }, { "id": "26edfda6-e331-4887-b91e-ac1735578b14", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-06T13:17:14.060Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=9381", "content-type": "text/plain; charset=utf-8", "date": "Thu, 06 Aug 2026 13:17:14 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c6650b8d7911755d0db0566b10abd41ae0147b15bc900c9a47e13a77e9b829d3", "previousChainHash": "7fc5682cf7f71b98deea05f93512a27e4ba9cd64aa73bc9a75ac226ff441c395" }, { "id": "4dde65f9-84db-468e-90f6-22c990a9c359", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-05T13:17:16.045Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Wed, 05 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "4dde65f9-84db-468e-90f6-22c990a9c359", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "902b323b0abe1661a07f32bcdef098f3a7a601efda9cd2628f5c8f6cdeeda64e", "previousChainHash": "2554f19637f59aee4f95f4decc283f8c7fb81460fbc320480786fe590d5770aa" }, { "id": "76a06035-aada-486d-b01b-69ba822709da", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-05T13:17:13.351Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=1166", "content-type": "text/plain; charset=utf-8", "date": "Wed, 05 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7fc5682cf7f71b98deea05f93512a27e4ba9cd64aa73bc9a75ac226ff441c395", "previousChainHash": "bf58c57481dba431bd699d8517f5a484bc61fb41856fc2b96911377b1eea2fe0" }, { "id": "116cf184-7323-4ba2-8c39-e56e6e686772", "domain": "oreilly.com", "signal": "llms", "requestedUrl": "https://oreilly.com/llms.txt", "finalUrl": "https://oreilly.com/llms.txt", "fetchedAt": "2026-08-04T13:17:16.747Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=2592000", "content-length": "240", "content-type": "text/html; charset=iso-8859-1", "date": "Tue, 04 Aug 2026 13:17:16 GMT", "location": "https://www.oreilly.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "116cf184-7323-4ba2-8c39-e56e6e686772", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2554f19637f59aee4f95f4decc283f8c7fb81460fbc320480786fe590d5770aa", "previousChainHash": null }, { "id": "731994fb-7ec4-474b-a247-1face4bc619d", "domain": "oreilly.com", "signal": "robots", "requestedUrl": "https://oreilly.com/robots.txt", "finalUrl": "https://www.oreilly.com/robots.txt", "fetchedAt": "2026-08-04T13:17:13.581Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=5583", "content-type": "text/plain; charset=utf-8", "date": "Tue, 04 Aug 2026 13:17:13 GMT", "last-modified": "Wed, 22 Jul 2026 20:55:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "41e41a4f7842aea7f91a92ba91ab7a89bb6e540aa16c2d5725851833eb28bf44", "artifactBytes": 2951, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "LinkedInBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 23, "pattern": "/images/" }, { "directive": "disallow", "line": 24, "pattern": "/graphics/" }, { "directive": "disallow", "line": 25, "pattern": "/admin/" }, { "directive": "disallow", "line": 26, "pattern": "/promos/" }, { "directive": "disallow", "line": 27, "pattern": "/ddp/" }, { "directive": "disallow", "line": 28, "pattern": "/dpp/" }, { "directive": "disallow", "line": 29, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 30, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 31, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 32, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 33, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 34, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 35, "pattern": "/cs/" }, { "directive": "disallow", "line": 36, "pattern": "/test/" }, { "directive": "disallow", "line": 37, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 38, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 39, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 40, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 41, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "Baiduspider", "PetalBot", "yisouspider", "008" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ] }, { "agents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.oreilly.com/sitemap_index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/images/" }, { "directive": "disallow", "line": 3, "pattern": "/graphics/" }, { "directive": "disallow", "line": 4, "pattern": "/admin/" }, { "directive": "disallow", "line": 5, "pattern": "/promos/" }, { "directive": "disallow", "line": 6, "pattern": "/ddp/" }, { "directive": "disallow", "line": 7, "pattern": "/dpp/" }, { "directive": "disallow", "line": 8, "pattern": "/programming/free/files/" }, { "directive": "disallow", "line": 9, "pattern": "/design/free/files/" }, { "directive": "disallow", "line": 10, "pattern": "/iot/free/files/" }, { "directive": "disallow", "line": 11, "pattern": "/data/free/files/" }, { "directive": "disallow", "line": 12, "pattern": "/webops-perf/free/files/" }, { "directive": "disallow", "line": 13, "pattern": "/web-platform/free/files/" }, { "directive": "disallow", "line": 14, "pattern": "/cs/" }, { "directive": "disallow", "line": 15, "pattern": "/test/" }, { "directive": "disallow", "line": 16, "pattern": "/*/?ar" }, { "directive": "disallow", "line": 17, "pattern": "/*/?orpq" }, { "directive": "disallow", "line": 18, "pattern": "/*/?discount=learn" }, { "directive": "disallow", "line": 19, "pattern": "/self-registration/*" }, { "directive": "disallow", "line": 20, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot", "anthropic-ai", "Google-Extended", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "rules": [ { "directive": "allow", "line": 80, "pattern": "/" }, { "directive": "disallow", "line": 81, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "rules": [ { "directive": "allow", "line": 59, "pattern": "/" }, { "directive": "disallow", "line": 60, "pattern": "/member/login/*?next=" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "731994fb-7ec4-474b-a247-1face4bc619d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bf58c57481dba431bd699d8517f5a484bc61fb41856fc2b96911377b1eea2fe0", "previousChainHash": null } ] }