{ "recordsNotConclusions": true, "registryVersion": "2026-09-20", "landscape": { "observedDomains": 20, "robotsRecords": 20, "contentSignalDomains": [ "cloudflare.com" ], "agents": { "GPTBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "GPTBot" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "GPTBot" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "disallowed", "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "disallowed", "matchedAgents": [ "GPTBot", "anthropic-ai", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "GPTBot" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "GPTBot" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "disallowed", "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "OAI-SearchBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "Google-Extended", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "partial", "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "OAI-AdsBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "ChatGPT-User": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "ChatGPT-User" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "ChatGPT-User" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "ChatGPT-User" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "disallowed", "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "ClaudeBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "ClaudeBot" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "disallowed", "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "Google-Extended", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "ClaudeBot" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "disallowed", "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "ClaudeBot" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "disallowed", "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "Claude-SearchBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "Claude-SearchBot" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "Google-Extended", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "Claude-SearchBot" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "disallowed", "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "Claude-SearchBot" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "partial", "matchedAgents": [ "Claude-SearchBot", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "Claude-User": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "Claude-User" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "Claude-User" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "disallowed", "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "Claude-User" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "Google-Extended": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "Google-Extended" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "Google-Extended", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "partial", "matchedAgents": [ "Google-Extended" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "Google-Extended" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "disallowed", "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "Applebot-Extended": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "Applebot-Extended" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "disallowed", "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ClaudeBot", "claude-web", "Claude-SearchBot", "YouBot", "Applebot-Extended", "MistralAI-User", "meta-webindexer", "GoogleAgent-Mariner", "Google-Extended", "DotBot", "archive.org_bot", "Pinterestbot", "TelegramBot", "kakaotalk-scrap", "getstream.io/opengraph-bot", "RamblerMail", "Brave", "OAI-SearchBot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "Applebot-Extended" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "disallowed", "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "Applebot-Extended" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "PerplexityBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "PerplexityBot" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "PerplexityBot" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "disallowed", "matchedAgents": [ "PerplexityBot" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "disallowed", "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "PerplexityBot" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "disallowed", "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "Perplexity-User": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "partial", "matchedAgents": [ "ChatGPT-User", "ChatGPT-User/2.0", "PerplexityBot", "Perplexity-User", "DuckAssistBot", "Applebot", "bingbot", "Googlebot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "Perplexity-User" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ], "CCBot": [ { "domain": "anthropic.com", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T04:17:05.742Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apnews.com", "state": "disallowed", "matchedAgents": [ "CCBot" ], "lastObservedAt": "2026-09-27T15:17:02.190Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "apple.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T06:17:01.851Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "cloudflare.com", "state": "allowed", "matchedAgents": [ "CCBot" ], "lastObservedAt": "2026-09-28T02:17:04.251Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "commoncrawl.org", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T08:17:02.666Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "google.com", "state": "partial", "matchedAgents": [ "*", "Yandex" ], "lastObservedAt": "2026-09-27T07:17:02.393Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "medium.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T11:17:02.392Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "openai.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T03:17:10.519Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "oreilly.com", "state": "disallowed", "matchedAgents": [ "GPTBot", "anthropic-ai", "cohere-ai", "CCBot", "AI2Bot", "Amazonbot", "Amazonbot-Video", "Bytespider", "meta-externalagent", "Diffbot", "omgili", "TimpiBot", "SeznamBot", "Exabot", "YandexBot", "Sogou", "360Spider", "YisouSpider", "Baiduspider-Render/2.0", "MJ12bot", "DataForSeoBot", "AliyunSecBot", "ArchiveTeam", "ArchiveTeam ArchiveBot", "AwarioBot", "ZoominfoBot", "ali-implementer", "Blueno", "BIGO-baiguoyuan", "WorksOgCrawler", "OnPageBot", "DoCoMo", "SAMSUNG-SGH-E250", "TA-Googlebot" ], "lastObservedAt": "2026-09-27T13:17:02.544Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "perplexity.ai", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T05:17:03.668Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "quora.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T12:17:02.579Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "reddit.com", "state": "disallowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T09:17:02.871Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslcollective.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T01:17:03.844Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "rslstandard.org", "state": "allowed", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-28T00:17:03.413Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "stackoverflow.com", "state": "unverified", "matchedAgents": [], "lastObservedAt": "2026-09-27T17:17:02.530Z", "parserVersion": null }, { "domain": "theguardian.com", "state": "disallowed", "matchedAgents": [ "NewsNow", "CCBot", "TurnitinBot", "PetalBot", "MoodleBot", "FacebookBot", "Bytespider", "Mojeek", "JenkersBot", "Seekr", "YouBot", "Arquivo-web-crawler", "coccocbot-web", "SeznamBot", "PerplexityBot", "yacy", "anthropic-ai", "ClaudeBot", "Claude-SearchBot", "Claude-User", "AwarioRssBot", "AwarioSmartBot", "SentiOne", "ImageSift", "Applebot-Extended", "YandexAdditional", "YandexAdditionalBot", "scalepostAI", "Buck", "meta-externalagent", "Amazonbot", "amazon-QBusiness", "DuckAssistBot", "Google-CloudVertexBot", "Amzn-SearchBot", "AhrefsBot", "AhrefsSiteAudit" ], "lastObservedAt": "2026-09-27T16:17:02.712Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "usatoday.com", "state": "disallowed", "matchedAgents": [ "CCBot" ], "lastObservedAt": "2026-09-27T19:17:03.438Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "voxmedia.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T18:17:01.994Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "yahoo.com", "state": "disallowed", "matchedAgents": [ "ADmantX", "AlphaBot", "anthropic-ai", "AwarioRssBot", "AwarioSmartBot", "BLEXBot", "Buzzbot", "Bytespider", "CCBot", "ChatGPT-User", "claritybot", "Claude-Web", "ClaudeBot", "cohere-ai", "Diffbot", "FacebookBot", "FriendlyCrawler", "Google-Extended", "GPTBot", "huggingface", "ImagesiftBot", "img2dataset", "magpie-crawler", "Meltwater", "Neevabot", "news-please", "NewsNow", "Nutch", "omgili", "omgilibot", "panscient.com", "Perplexity-ai", "PerplexityBot", "PetalBot", "PiplBot", "scoop.it", "Scrapy", "Seekr", "SentiBot", "SeznamBot", "TurnitinBot", "YouBot", "ZumBot" ], "lastObservedAt": "2026-09-27T10:17:03.356Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" }, { "domain": "ziffdavis.com", "state": "partial", "matchedAgents": [ "*" ], "lastObservedAt": "2026-09-27T14:17:02.655Z", "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20" } ] } } }