{
  "recordsNotConclusions": true,
  "registryVersion": "2026-09-20",
  "agent": {
    "slug": "google-extended",
    "token": "Google-Extended",
    "operator": "Google",
    "purpose": "mixed",
    "isFetcher": false,
    "description": "Google's robots.txt control token for Gemini model training and grounding uses of content already crawled by Google.",
    "sourceUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
    "sourceCheckedAt": "2026-09-20",
    "notes": "Control token only; Google says it is not a separate HTTP fetcher and does not affect Google Search inclusion or ranking."
  },
  "observations": [
    {
      "domain": "anthropic.com",
      "state": "allowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T04:17:02.963Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "apnews.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T15:17:02.190Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "apple.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T06:17:01.851Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "cloudflare.com",
      "state": "allowed",
      "matchedAgents": [
        "Google-Extended"
      ],
      "lastObservedAt": "2026-09-27T02:17:02.247Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "commoncrawl.org",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T08:17:02.666Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "google.com",
      "state": "partial",
      "matchedAgents": [
        "*",
        "Yandex"
      ],
      "lastObservedAt": "2026-09-27T07:17:02.393Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "medium.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T11:17:02.392Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "openai.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T03:17:03.795Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "oreilly.com",
      "state": "partial",
      "matchedAgents": [
        "ClaudeBot",
        "claude-web",
        "Claude-SearchBot",
        "YouBot",
        "Applebot-Extended",
        "MistralAI-User",
        "meta-webindexer",
        "GoogleAgent-Mariner",
        "Google-Extended",
        "DotBot",
        "archive.org_bot",
        "Pinterestbot",
        "TelegramBot",
        "kakaotalk-scrap",
        "getstream.io/opengraph-bot",
        "RamblerMail",
        "Brave",
        "OAI-SearchBot"
      ],
      "lastObservedAt": "2026-09-27T13:17:02.544Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "perplexity.ai",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T05:17:02.211Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "quora.com",
      "state": "partial",
      "matchedAgents": [
        "Google-Extended"
      ],
      "lastObservedAt": "2026-09-27T12:17:02.579Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "reddit.com",
      "state": "disallowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T09:17:02.871Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "rslcollective.org",
      "state": "allowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T01:17:02.879Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "rslstandard.org",
      "state": "allowed",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T00:17:02.611Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "stackoverflow.com",
      "state": "unverified",
      "matchedAgents": [],
      "lastObservedAt": "2026-09-27T17:17:02.530Z",
      "parserVersion": null
    },
    {
      "domain": "theguardian.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T16:17:02.712Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "usatoday.com",
      "state": "disallowed",
      "matchedAgents": [
        "Google-Extended"
      ],
      "lastObservedAt": "2026-09-27T19:17:03.438Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "voxmedia.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T18:17:01.994Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "yahoo.com",
      "state": "disallowed",
      "matchedAgents": [
        "ADmantX",
        "AlphaBot",
        "anthropic-ai",
        "AwarioRssBot",
        "AwarioSmartBot",
        "BLEXBot",
        "Buzzbot",
        "Bytespider",
        "CCBot",
        "ChatGPT-User",
        "claritybot",
        "Claude-Web",
        "ClaudeBot",
        "cohere-ai",
        "Diffbot",
        "FacebookBot",
        "FriendlyCrawler",
        "Google-Extended",
        "GPTBot",
        "huggingface",
        "ImagesiftBot",
        "img2dataset",
        "magpie-crawler",
        "Meltwater",
        "Neevabot",
        "news-please",
        "NewsNow",
        "Nutch",
        "omgili",
        "omgilibot",
        "panscient.com",
        "Perplexity-ai",
        "PerplexityBot",
        "PetalBot",
        "PiplBot",
        "scoop.it",
        "Scrapy",
        "Seekr",
        "SentiBot",
        "SeznamBot",
        "TurnitinBot",
        "YouBot",
        "ZumBot"
      ],
      "lastObservedAt": "2026-09-27T10:17:03.356Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    },
    {
      "domain": "ziffdavis.com",
      "state": "partial",
      "matchedAgents": [
        "*"
      ],
      "lastObservedAt": "2026-09-27T14:17:02.655Z",
      "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20"
    }
  ]
}