{
  "domain": "yahoo.com",
  "recordsNotConclusions": true,
  "observations": [
    {
      "id": "6b31cae4-eb68-48fc-926a-76a25f1b496c",
      "domain": "yahoo.com",
      "signal": "llms",
      "requestedUrl": "https://yahoo.com/llms.txt",
      "finalUrl": "https://yahoo.com/llms.txt",
      "fetchedAt": "2026-08-13T10:17:16.587Z",
      "httpStatus": 301,
      "responseHeaders": {
        "cache-control": "no-store, no-cache",
        "content-length": "8",
        "content-type": "text/html",
        "date": "Thu, 13 Aug 2026 10:17:16 GMT",
        "location": "https://www.yahoo.com/llms.txt",
        "server": "cloudflare"
      },
      "parseStatus": "http_error",
      "changed": false,
      "artifactSha256": null,
      "artifactBytes": null,
      "parserVersion": null,
      "parsed": null,
      "errorCode": "redirect_cross_host",
      "canonicalEventId": "6b31cae4-eb68-48fc-926a-76a25f1b496c",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "4c01813e4c627ae3dafef9c7666c134f88a3063b50c6ff6ae5e3dc5ed1ad8085",
      "previousChainHash": "edfe5eaa1da526671d7f64c5832f87d096331233908c6b73333cded3cd2b7f02"
    },
    {
      "id": "c940a12f-1489-45ed-bbc8-73cba10bd0a3",
      "domain": "yahoo.com",
      "signal": "robots",
      "requestedUrl": "https://yahoo.com/robots.txt",
      "finalUrl": "https://www.yahoo.com/robots.txt",
      "fetchedAt": "2026-08-13T10:17:13.627Z",
      "httpStatus": 200,
      "responseHeaders": {
        "cache-control": "max-age=31536000, public",
        "content-length": "1624",
        "content-type": "text/plain",
        "date": "Thu, 13 Aug 2026 10:17:13 GMT",
        "etag": "\"2073952842e49b8ada21fe7ed0244f3a\"",
        "last-modified": "Tue, 31 Mar 2026 17:22:50 GMT",
        "server": "cloudflare"
      },
      "parseStatus": "parsed",
      "changed": false,
      "artifactSha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9",
      "artifactBytes": 1624,
      "parserVersion": "robots-rfc9309-rsl-0.1.0",
      "parsed": {
        "globalLicenses": [],
        "groups": [
          {
            "agents": [
              "*"
            ],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 2,
                "pattern": "/info/p.gif"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/p/"
              },
              {
                "directive": "disallow",
                "line": 4,
                "pattern": "/r/"
              },
              {
                "directive": "disallow",
                "line": 5,
                "pattern": "/bin/"
              },
              {
                "directive": "disallow",
                "line": 6,
                "pattern": "/caas/"
              },
              {
                "directive": "disallow",
                "line": 7,
                "pattern": "/blank.html"
              },
              {
                "directive": "disallow",
                "line": 8,
                "pattern": "/includes/"
              },
              {
                "directive": "disallow",
                "line": 9,
                "pattern": "/_td_api"
              },
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/tdv2_fp"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/nel_ms"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/fp_ms"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/sports_fp_ms"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/search_ms"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/_tdpp_api"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/_remote"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/_multiremote"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/_tdhl_api"
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/digest"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/fpjs"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/myjs"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/news/m/"
              }
            ]
          },
          {
            "agents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ]
          },
          {
            "agents": [
              "Claude-SearchBot",
              "OAI-SearchBot"
            ],
            "crawlDelaySeconds": null,
            "licenses": [],
            "rules": [
              {
                "directive": "disallow",
                "line": 71,
                "pattern": "*/articles/"
              }
            ]
          }
        ],
        "parserVersion": "robots-rfc9309-rsl-0.1.0",
        "registryVersion": "2026-08-03",
        "sitemaps": [
          "https://www.yahoo.com/news/weather/sitemap.xml",
          "https://www.yahoo.com/news-sitemap-index.xml",
          "https://www.yahoo.com/sitemap-index.xml"
        ],
        "trackedAgents": {
          "Applebot-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 2,
                "pattern": "/info/p.gif"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/p/"
              },
              {
                "directive": "disallow",
                "line": 4,
                "pattern": "/r/"
              },
              {
                "directive": "disallow",
                "line": 5,
                "pattern": "/bin/"
              },
              {
                "directive": "disallow",
                "line": 6,
                "pattern": "/caas/"
              },
              {
                "directive": "disallow",
                "line": 7,
                "pattern": "/blank.html"
              },
              {
                "directive": "disallow",
                "line": 8,
                "pattern": "/includes/"
              },
              {
                "directive": "disallow",
                "line": 9,
                "pattern": "/_td_api"
              },
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/tdv2_fp"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/nel_ms"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/fp_ms"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/sports_fp_ms"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/search_ms"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/_tdpp_api"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/_remote"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/_multiremote"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/_tdhl_api"
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/digest"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/fpjs"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/myjs"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/news/m/"
              }
            ],
            "state": "partial"
          },
          "CCBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "ChatGPT-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "Claude-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "Claude-SearchBot",
              "OAI-SearchBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 71,
                "pattern": "*/articles/"
              }
            ],
            "state": "partial"
          },
          "Claude-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 2,
                "pattern": "/info/p.gif"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/p/"
              },
              {
                "directive": "disallow",
                "line": 4,
                "pattern": "/r/"
              },
              {
                "directive": "disallow",
                "line": 5,
                "pattern": "/bin/"
              },
              {
                "directive": "disallow",
                "line": 6,
                "pattern": "/caas/"
              },
              {
                "directive": "disallow",
                "line": 7,
                "pattern": "/blank.html"
              },
              {
                "directive": "disallow",
                "line": 8,
                "pattern": "/includes/"
              },
              {
                "directive": "disallow",
                "line": 9,
                "pattern": "/_td_api"
              },
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/tdv2_fp"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/nel_ms"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/fp_ms"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/sports_fp_ms"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/search_ms"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/_tdpp_api"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/_remote"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/_multiremote"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/_tdhl_api"
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/digest"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/fpjs"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/myjs"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/news/m/"
              }
            ],
            "state": "partial"
          },
          "ClaudeBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "Google-Extended": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "GPTBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          },
          "OAI-SearchBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "Claude-SearchBot",
              "OAI-SearchBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 71,
                "pattern": "*/articles/"
              }
            ],
            "state": "partial"
          },
          "Perplexity-User": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "*"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 2,
                "pattern": "/info/p.gif"
              },
              {
                "directive": "disallow",
                "line": 3,
                "pattern": "/p/"
              },
              {
                "directive": "disallow",
                "line": 4,
                "pattern": "/r/"
              },
              {
                "directive": "disallow",
                "line": 5,
                "pattern": "/bin/"
              },
              {
                "directive": "disallow",
                "line": 6,
                "pattern": "/caas/"
              },
              {
                "directive": "disallow",
                "line": 7,
                "pattern": "/blank.html"
              },
              {
                "directive": "disallow",
                "line": 8,
                "pattern": "/includes/"
              },
              {
                "directive": "disallow",
                "line": 9,
                "pattern": "/_td_api"
              },
              {
                "directive": "disallow",
                "line": 10,
                "pattern": "/tdv2_fp"
              },
              {
                "directive": "disallow",
                "line": 11,
                "pattern": "/nel_ms"
              },
              {
                "directive": "disallow",
                "line": 12,
                "pattern": "/fp_ms"
              },
              {
                "directive": "disallow",
                "line": 13,
                "pattern": "/sports_fp_ms"
              },
              {
                "directive": "disallow",
                "line": 14,
                "pattern": "/search_ms"
              },
              {
                "directive": "disallow",
                "line": 15,
                "pattern": "/_tdpp_api"
              },
              {
                "directive": "disallow",
                "line": 16,
                "pattern": "/_remote"
              },
              {
                "directive": "disallow",
                "line": 17,
                "pattern": "/_multiremote"
              },
              {
                "directive": "disallow",
                "line": 18,
                "pattern": "/_tdhl_api"
              },
              {
                "directive": "disallow",
                "line": 19,
                "pattern": "/digest"
              },
              {
                "directive": "disallow",
                "line": 20,
                "pattern": "/fpjs"
              },
              {
                "directive": "disallow",
                "line": 21,
                "pattern": "/myjs"
              },
              {
                "directive": "disallow",
                "line": 22,
                "pattern": "/news/m/"
              }
            ],
            "state": "partial"
          },
          "PerplexityBot": {
            "crawlDelaySeconds": null,
            "licenses": [],
            "matchedAgents": [
              "ADmantX",
              "AlphaBot",
              "anthropic-ai",
              "AwarioRssBot",
              "AwarioSmartBot",
              "BLEXBot",
              "Buzzbot",
              "Bytespider",
              "CCBot",
              "ChatGPT-User",
              "claritybot",
              "Claude-Web",
              "ClaudeBot",
              "cohere-ai",
              "Diffbot",
              "FacebookBot",
              "FriendlyCrawler",
              "Google-Extended",
              "GPTBot",
              "huggingface",
              "ImagesiftBot",
              "img2dataset",
              "magpie-crawler",
              "Meltwater",
              "Neevabot",
              "news-please",
              "NewsNow",
              "Nutch",
              "omgili",
              "omgilibot",
              "panscient.com",
              "Perplexity-ai",
              "PerplexityBot",
              "PetalBot",
              "PiplBot",
              "scoop.it",
              "Scrapy",
              "Seekr",
              "SentiBot",
              "SeznamBot",
              "TurnitinBot",
              "YouBot",
              "ZumBot"
            ],
            "rules": [
              {
                "directive": "disallow",
                "line": 67,
                "pattern": "/"
              }
            ],
            "state": "disallowed"
          }
        },
        "warnings": []
      },
      "errorCode": null,
      "canonicalEventId": "4de48932-ffd3-467a-94eb-2d6fcfb8f380",
      "ledgerFormatVersion": "crawlledger-event-0.1.0",
      "chainHash": "ab48b9383fedeebce8cfa219ccd55ad042b14c0f0cf7cc586f5522d085190de1",
      "previousChainHash": "747f4d0bac2d773dbac0df06e94638757ffbdc189ae67bf1f7f19737b0d1450d"
    }
  ],
  "payPerCrawl": {
    "status": "unverified",
    "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API."
  }
}