{ "domain": "yahoo.com", "recordsNotConclusions": true, "events": [ { "id": "6b31cae4-eb68-48fc-926a-76a25f1b496c", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-13T10:17:16.587Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Thu, 13 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "6b31cae4-eb68-48fc-926a-76a25f1b496c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "4c01813e4c627ae3dafef9c7666c134f88a3063b50c6ff6ae5e3dc5ed1ad8085", "previousChainHash": "edfe5eaa1da526671d7f64c5832f87d096331233908c6b73333cded3cd2b7f02" }, { "id": "c940a12f-1489-45ed-bbc8-73cba10bd0a3", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-13T10:17:13.627Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Thu, 13 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ab48b9383fedeebce8cfa219ccd55ad042b14c0f0cf7cc586f5522d085190de1", "previousChainHash": "747f4d0bac2d773dbac0df06e94638757ffbdc189ae67bf1f7f19737b0d1450d" }, { "id": "93c62733-e8af-4d26-852a-fa3ccd90dc0c", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-12T10:17:16.266Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Wed, 12 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "93c62733-e8af-4d26-852a-fa3ccd90dc0c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "edfe5eaa1da526671d7f64c5832f87d096331233908c6b73333cded3cd2b7f02", "previousChainHash": "825bb7276b6c09c3f5c4911b9fa68ab36bf8a559dc44efe4a88f1d5539abaafd" }, { "id": "2fd4cf0d-9182-4da7-87eb-05a3ddb5467e", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-12T10:17:13.348Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Wed, 12 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "747f4d0bac2d773dbac0df06e94638757ffbdc189ae67bf1f7f19737b0d1450d", "previousChainHash": "942abb32dcb532f32ce1f5e5d5f74f44c393113771a482e36303dd8017371030" }, { "id": "a6e7949b-500e-4a9d-b6b6-3026edc0636f", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-11T10:17:15.808Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Tue, 11 Aug 2026 10:17:15 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "a6e7949b-500e-4a9d-b6b6-3026edc0636f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "825bb7276b6c09c3f5c4911b9fa68ab36bf8a559dc44efe4a88f1d5539abaafd", "previousChainHash": "91fc9c66f315108b749b8b734ef9e8500a4802e8967dcd78fa7ba843fa8cfdb8" }, { "id": "be190654-85b3-4715-b725-62b78a4aa5ae", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-11T10:17:12.972Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Tue, 11 Aug 2026 10:17:12 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "942abb32dcb532f32ce1f5e5d5f74f44c393113771a482e36303dd8017371030", "previousChainHash": "43fe918bb62d7febf6bb7893b95aaa59e8197bd493bbbb720a3d94c446cc4603" }, { "id": "34c97c17-acdf-4720-bc7a-a938676f633c", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-10T10:17:16.538Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Mon, 10 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "34c97c17-acdf-4720-bc7a-a938676f633c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "91fc9c66f315108b749b8b734ef9e8500a4802e8967dcd78fa7ba843fa8cfdb8", "previousChainHash": "bc033af8c67550b322fe2acddb1f12b4fb9e23aff13c55bedd7dfcd520784694" }, { "id": "cc30ea37-09b8-4de3-9631-748df736adb9", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-10T10:17:13.687Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Mon, 10 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "43fe918bb62d7febf6bb7893b95aaa59e8197bd493bbbb720a3d94c446cc4603", "previousChainHash": "bdada4001db58d55b6233f6104c10f163a7911c9707b8cbe9ab83ea8cdaa03df" }, { "id": "7f25a25a-44ef-4bae-bc5d-3b9fbcedcb92", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-09T10:17:16.175Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Sun, 09 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "7f25a25a-44ef-4bae-bc5d-3b9fbcedcb92", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bc033af8c67550b322fe2acddb1f12b4fb9e23aff13c55bedd7dfcd520784694", "previousChainHash": "eac5cbacbbcc8459fd49ae288ea63a6db2664f6328288f2ec1365e6d44cbb47d" }, { "id": "8c6d178b-3582-48c4-af44-d80d29a01536", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-09T10:17:13.412Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Sun, 09 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bdada4001db58d55b6233f6104c10f163a7911c9707b8cbe9ab83ea8cdaa03df", "previousChainHash": "60bce38e0a79fc0aa44bd057b6d802a6a43aca3cd716833a4c31804f9eb01e28" }, { "id": "eb47bfdf-79d7-40ac-ae7a-e77a6366db98", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-08T10:17:16.659Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Sat, 08 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "eb47bfdf-79d7-40ac-ae7a-e77a6366db98", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "eac5cbacbbcc8459fd49ae288ea63a6db2664f6328288f2ec1365e6d44cbb47d", "previousChainHash": "575d2327b29f3d5381fe447958651438ecbfe72262705b9cdabee69022447897" }, { "id": "9878f457-869c-4f1a-add0-fdb5f637ed4f", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-08T10:17:13.644Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Sat, 08 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "60bce38e0a79fc0aa44bd057b6d802a6a43aca3cd716833a4c31804f9eb01e28", "previousChainHash": "52d79221f2a6268da39ddbb3656ac04d426a18c86d785e9b8599dd876038bf3a" }, { "id": "133f5f36-3c18-4fee-877d-e55c741c956a", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-07T10:17:16.482Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Fri, 07 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "133f5f36-3c18-4fee-877d-e55c741c956a", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "575d2327b29f3d5381fe447958651438ecbfe72262705b9cdabee69022447897", "previousChainHash": "0e40b59e199a8962c485d416f3debc1c51a080277ff199dd92442c2518f64c83" }, { "id": "018d5b6d-8917-4daa-81c3-219d25e3a136", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-07T10:17:13.618Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Fri, 07 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "52d79221f2a6268da39ddbb3656ac04d426a18c86d785e9b8599dd876038bf3a", "previousChainHash": "25962b9136805d023c43cc8c06d02fa932a3434a7912d1acccba00a09197033a" }, { "id": "2195bf5f-6807-4eb3-936b-0985dd033cf5", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-06T10:17:16.046Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Thu, 06 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "2195bf5f-6807-4eb3-936b-0985dd033cf5", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "0e40b59e199a8962c485d416f3debc1c51a080277ff199dd92442c2518f64c83", "previousChainHash": "375bb4e9a325472b6d4c6b8325ab9b000556e63cd09376f2b72cf722d3e8ea54" }, { "id": "10e20f3b-ce72-4ec4-85ce-08fbc5375398", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-06T10:17:13.242Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Thu, 06 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "25962b9136805d023c43cc8c06d02fa932a3434a7912d1acccba00a09197033a", "previousChainHash": "774a400e77a35bd793de8341fd3ab532b7be1d91087873b640b95022c4da659a" }, { "id": "a321d796-d474-4624-9064-9108e30a3072", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-05T10:17:16.330Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Wed, 05 Aug 2026 10:17:16 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "a321d796-d474-4624-9064-9108e30a3072", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "375bb4e9a325472b6d4c6b8325ab9b000556e63cd09376f2b72cf722d3e8ea54", "previousChainHash": "27303880e398beff2937a91380076d08296a9a845001ce80ddaada701bff5320" }, { "id": "ab1176ef-602d-4119-812e-950bd1a6f1fc", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-05T10:17:13.391Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Wed, 05 Aug 2026 10:17:13 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "774a400e77a35bd793de8341fd3ab532b7be1d91087873b640b95022c4da659a", "previousChainHash": "973e7634a3d1a1030eba77220c35f76742f79dfea44c54699f1bbc6e42c2e284" }, { "id": "0e45b816-8f33-4d75-96d0-aec0f3386c00", "domain": "yahoo.com", "signal": "llms", "requestedUrl": "https://yahoo.com/llms.txt", "finalUrl": "https://yahoo.com/llms.txt", "fetchedAt": "2026-08-04T10:17:23.409Z", "httpStatus": 301, "responseHeaders": { "cache-control": "no-store, no-cache", "content-length": "8", "content-type": "text/html", "date": "Tue, 04 Aug 2026 10:17:23 GMT", "location": "https://www.yahoo.com/llms.txt", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "0e45b816-8f33-4d75-96d0-aec0f3386c00", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "27303880e398beff2937a91380076d08296a9a845001ce80ddaada701bff5320", "previousChainHash": null }, { "id": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "domain": "yahoo.com", "signal": "robots", "requestedUrl": "https://yahoo.com/robots.txt", "finalUrl": "https://www.yahoo.com/robots.txt", "fetchedAt": "2026-08-04T10:17:14.397Z", "httpStatus": 200, "responseHeaders": { "cache-control": "max-age=31536000, public", "content-length": "1624", "content-type": "text/plain", "date": "Tue, 04 Aug 2026 10:17:14 GMT", "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"", "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "artifactBytes": 1624, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ] }, { "agents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot", "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.yahoo.com/news/weather/sitemap.xml", "https://www.yahoo.com/news-sitemap-index.xml", "https://www.yahoo.com/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "*/articles/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/info/p.gif" }, { "directive": "disallow", "line": 3, "pattern": "/p/" }, { "directive": "disallow", "line": 4, "pattern": "/r/" }, { "directive": "disallow", "line": 5, "pattern": "/bin/" }, { "directive": "disallow", "line": 6, "pattern": "/caas/" }, { "directive": "disallow", "line": 7, "pattern": "/blank.html" }, { "directive": "disallow", "line": 8, "pattern": "/includes/" }, { "directive": "disallow", "line": 9, "pattern": "/_td_api" }, { "directive": "disallow", "line": 10, "pattern": "/tdv2_fp" }, { "directive": "disallow", "line": 11, "pattern": "/nel_ms" }, { "directive": "disallow", "line": 12, "pattern": "/fp_ms" }, { "directive": "disallow", "line": 13, "pattern": "/sports_fp_ms" }, { "directive": "disallow", "line": 14, "pattern": "/search_ms" }, { "directive": "disallow", "line": 15, "pattern": "/_tdpp_api" }, { "directive": "disallow", "line": 16, "pattern": "/_remote" }, { "directive": "disallow", "line": 17, "pattern": "/_multiremote" }, { "directive": "disallow", "line": 18, "pattern": "/_tdhl_api" }, { "directive": "disallow", "line": 19, "pattern": "/digest" }, { "directive": "disallow", "line": 20, "pattern": "/fpjs" }, { "directive": "disallow", "line": 21, "pattern": "/myjs" }, { "directive": "disallow", "line": 22, "pattern": "/news/m/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "973e7634a3d1a1030eba77220c35f76742f79dfea44c54699f1bbc6e42c2e284", "previousChainHash": null } ] }