{ "domain": "apnews.com", "recordsNotConclusions": true, "events": [ { "id": "d45a1e85-10d1-409c-8c0a-604830d5d616", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-11T15:17:06.467Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Wed, 11 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "d45a1e85-10d1-409c-8c0a-604830d5d616", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f8ee2352fe44c828d2f5af9f2428fee0b4c50112e55a12b79d7564f9fb0eb677", "previousChainHash": "c93a09ac351fbb634f5fa1d097f6e2c468cdd7ef19a82456c55d29dd0af3cab0" }, { "id": "5f881f9a-cb80-46fc-9fba-08d5ae593a89", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-11T15:17:03.284Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Wed, 11 Nov 2026 15:17:03 GMT", "last-modified": "Wed, 11 Nov 2026 15:01:01 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e7d4f45cd1c2f8bdd12eb58905549d143fa6c7f472a526c33f6c6765a4604f9d", "previousChainHash": "565267f552f5d7087dc06906b708b438b91b853eafad2152fb3c14930f3d6235" }, { "id": "dcc44f3d-8787-4388-94b3-6efc6837fe85", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-10T15:17:06.850Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Tue, 10 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "dcc44f3d-8787-4388-94b3-6efc6837fe85", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c93a09ac351fbb634f5fa1d097f6e2c468cdd7ef19a82456c55d29dd0af3cab0", "previousChainHash": "f604052b40a909804c982c608895ba9b56cd3b4b458e97c5b0de05a3405dc1ed" }, { "id": "842abc68-fdf8-46cb-9d43-cc7d1967a63f", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-10T15:17:03.529Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Tue, 10 Nov 2026 15:17:03 GMT", "last-modified": "Tue, 10 Nov 2026 15:16:48 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "565267f552f5d7087dc06906b708b438b91b853eafad2152fb3c14930f3d6235", "previousChainHash": "42586ffb1024281647e86e9ee2728a41c233cde03180440a30e5cda865191cb6" }, { "id": "e2ffc6cb-1f5f-49c7-93c5-0c3449506ea1", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-09T15:17:06.602Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Mon, 09 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "e2ffc6cb-1f5f-49c7-93c5-0c3449506ea1", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f604052b40a909804c982c608895ba9b56cd3b4b458e97c5b0de05a3405dc1ed", "previousChainHash": "36f2777becb9b8b6719aaaefe489bfaf0fccc188fdd9f421bea91f506b626360" }, { "id": "5d7c6b3a-405e-43b9-9709-d92292e09e4d", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-09T15:17:03.254Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Mon, 09 Nov 2026 15:17:03 GMT", "last-modified": "Mon, 09 Nov 2026 14:48:29 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "42586ffb1024281647e86e9ee2728a41c233cde03180440a30e5cda865191cb6", "previousChainHash": "debcfb6aabcf54e036d82de072effa33fa87075f5c0304a23b5080324668d3b3" }, { "id": "063cf872-1e23-4644-b4e1-3e41061b7c58", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-08T15:17:05.880Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sun, 08 Nov 2026 15:17:05 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "063cf872-1e23-4644-b4e1-3e41061b7c58", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "36f2777becb9b8b6719aaaefe489bfaf0fccc188fdd9f421bea91f506b626360", "previousChainHash": "f0fb501be0622cd86e6f8e6f3fe009452f370a4d0981ed46b269b1e76f9e5f22" }, { "id": "31958251-eddc-41d2-867a-50f25af66d20", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-08T15:17:02.874Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sun, 08 Nov 2026 15:17:02 GMT", "last-modified": "Sun, 08 Nov 2026 14:44:41 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "debcfb6aabcf54e036d82de072effa33fa87075f5c0304a23b5080324668d3b3", "previousChainHash": "1a8f176a314ce1773042ca6d8f12ec9b3f0fa97810d27719c9a5637cefa8f177" }, { "id": "49a78790-9b7c-4c0c-b57b-3ab50e57990f", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-07T15:17:05.487Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sat, 07 Nov 2026 15:17:05 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "49a78790-9b7c-4c0c-b57b-3ab50e57990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f0fb501be0622cd86e6f8e6f3fe009452f370a4d0981ed46b269b1e76f9e5f22", "previousChainHash": "0d20001e477d5efb4a9f89277b1087212d43b44e42e19364f6d3f1d3e9d3f94b" }, { "id": "99348d03-7853-4132-8b8d-c917e7563458", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-07T15:17:02.430Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sat, 07 Nov 2026 15:17:02 GMT", "last-modified": "Sat, 07 Nov 2026 15:17:02 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1a8f176a314ce1773042ca6d8f12ec9b3f0fa97810d27719c9a5637cefa8f177", "previousChainHash": "c39f6e2e720157974b08da4041ac0328dca67003ec0f938ce2d1ecb8d14e84a4" }, { "id": "aa9bc721-e0cf-4f98-8d51-2b7679643163", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-06T15:17:06.755Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Fri, 06 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "aa9bc721-e0cf-4f98-8d51-2b7679643163", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "0d20001e477d5efb4a9f89277b1087212d43b44e42e19364f6d3f1d3e9d3f94b", "previousChainHash": "687c351df6e5023e7af1a2c890d217fabeef7f86b0349154efd175d1f6693b66" }, { "id": "79bc8d26-c91c-4804-a0a3-8daf5d20bf2c", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-06T15:17:03.474Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Fri, 06 Nov 2026 15:17:03 GMT", "last-modified": "Fri, 06 Nov 2026 15:15:55 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c39f6e2e720157974b08da4041ac0328dca67003ec0f938ce2d1ecb8d14e84a4", "previousChainHash": "caea4edd72d7051bd40556cc231bfda384b87e1bfc343973d388a4ff9b3137e8" }, { "id": "70c70832-e2d6-4117-93aa-a279abac0fa6", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-05T15:17:07.309Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Thu, 05 Nov 2026 15:17:07 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "70c70832-e2d6-4117-93aa-a279abac0fa6", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "687c351df6e5023e7af1a2c890d217fabeef7f86b0349154efd175d1f6693b66", "previousChainHash": "b3affe81f90b09f4352183fda6b6ea549cebf2d0963a342ad9ab82ce372b8eaf" }, { "id": "b1d23373-e077-4aaf-8c12-c9834563990f", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-05T15:17:03.458Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Thu, 05 Nov 2026 15:17:03 GMT", "last-modified": "Thu, 05 Nov 2026 15:17:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "e5c414b02cc41292c7c8ae3d1175bfc94d7a73e98e27e5ccfc9bd7adfcb35c27", "artifactBytes": 1419, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 74, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b1d23373-e077-4aaf-8c12-c9834563990f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "caea4edd72d7051bd40556cc231bfda384b87e1bfc343973d388a4ff9b3137e8", "previousChainHash": "44c6f2907c694f51288fb0531461620ce83c4107da57477417fe7ab158d10f65" }, { "id": "974e45ba-8d26-4f76-b600-c6a7eda98fcf", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-04T15:17:08.240Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Wed, 04 Nov 2026 15:17:08 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "974e45ba-8d26-4f76-b600-c6a7eda98fcf", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "b3affe81f90b09f4352183fda6b6ea549cebf2d0963a342ad9ab82ce372b8eaf", "previousChainHash": "6bc3c68caf9ad3da442898bca9c4fe88c62d5a45bf17d2b7c7cdf976d39ae582" }, { "id": "3b1de8a6-8a17-450a-99e7-2d8480ea3c70", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-04T15:17:05.477Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Wed, 04 Nov 2026 15:17:05 GMT", "last-modified": "Wed, 04 Nov 2026 14:40:43 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "44c6f2907c694f51288fb0531461620ce83c4107da57477417fe7ab158d10f65", "previousChainHash": "237a94a594a7badb5dd4c141322da990fcd8e2c46a5ef614bea0faa0314554a2" }, { "id": "1f320945-a7aa-474e-b98c-c7309b896391", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-03T15:17:06.518Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Tue, 03 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "1f320945-a7aa-474e-b98c-c7309b896391", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6bc3c68caf9ad3da442898bca9c4fe88c62d5a45bf17d2b7c7cdf976d39ae582", "previousChainHash": "201e5d3049d28d680f83bd18084cf734a59df908fdb79511773263bf5730c09a" }, { "id": "5b1efaf1-5554-4aec-8e4f-3105f7378e1c", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-03T15:17:03.298Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Tue, 03 Nov 2026 15:17:03 GMT", "last-modified": "Tue, 03 Nov 2026 14:50:49 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "237a94a594a7badb5dd4c141322da990fcd8e2c46a5ef614bea0faa0314554a2", "previousChainHash": "c1ca5c026b5a62c88d2c7dcb0d63de25724aa0437cf31bb1e2ee3b18ae24a259" }, { "id": "a45d8504-800a-4054-8bab-83275c206e47", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-02T15:17:06.898Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Mon, 02 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "a45d8504-800a-4054-8bab-83275c206e47", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "201e5d3049d28d680f83bd18084cf734a59df908fdb79511773263bf5730c09a", "previousChainHash": "9592fea39a852852c3fa6319c469fb949c544a0093e2c9248574cf54aabf4115" }, { "id": "8d260aa2-670c-4589-9fe9-dcbe985c8284", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-02T15:17:03.164Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Mon, 02 Nov 2026 15:17:03 GMT", "last-modified": "Mon, 02 Nov 2026 15:01:06 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c1ca5c026b5a62c88d2c7dcb0d63de25724aa0437cf31bb1e2ee3b18ae24a259", "previousChainHash": "ae0873a0f9ba2d999382c48172708790b01e707800f28304f10b9faf50108f48" }, { "id": "0e64b363-eac7-4b81-aaa5-d9810495bc48", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-11-01T15:17:06.090Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sun, 01 Nov 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "0e64b363-eac7-4b81-aaa5-d9810495bc48", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9592fea39a852852c3fa6319c469fb949c544a0093e2c9248574cf54aabf4115", "previousChainHash": "3c52c10450eda8e41547898673031dacc3f077a3d14113e1d789a800bd872b70" }, { "id": "1d6a2f03-0aa5-4574-8545-8a4c2208c9fa", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-11-01T15:17:03.198Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sun, 01 Nov 2026 15:17:03 GMT", "last-modified": "Sun, 01 Nov 2026 15:00:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ae0873a0f9ba2d999382c48172708790b01e707800f28304f10b9faf50108f48", "previousChainHash": "39c86de3f9b6dfc93296747a37552f20792a8b7adbc29dc9561d51039dec4e92" }, { "id": "abdeee34-e490-4841-b2be-e5355e82cd53", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-31T15:17:06.157Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sat, 31 Oct 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "abdeee34-e490-4841-b2be-e5355e82cd53", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3c52c10450eda8e41547898673031dacc3f077a3d14113e1d789a800bd872b70", "previousChainHash": "50de302a6fc1f57dff97ae18583c8340c9c029ee6365089888af29544fc2e12b" }, { "id": "fd5a96c2-ef98-4c7e-8287-6533d91b9ce4", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-31T15:17:03.022Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sat, 31 Oct 2026 15:17:03 GMT", "last-modified": "Sat, 31 Oct 2026 15:15:12 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "39c86de3f9b6dfc93296747a37552f20792a8b7adbc29dc9561d51039dec4e92", "previousChainHash": "29753ebd2a3ce94fcc404b98f969d788355a72dc588134d2f6bcbea28c3aeb46" }, { "id": "1a11a3ff-2141-4284-80ba-d1d98b4b9f55", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-30T15:17:06.243Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Fri, 30 Oct 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "1a11a3ff-2141-4284-80ba-d1d98b4b9f55", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "50de302a6fc1f57dff97ae18583c8340c9c029ee6365089888af29544fc2e12b", "previousChainHash": "e901993bfbe4b09ab2b7dbed2e802d6ffe98a0e0a5a3e744d61c0d735ad18a1a" }, { "id": "1b4986f6-7369-4cc3-a6c2-69e7f1011284", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-30T15:17:03.040Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Fri, 30 Oct 2026 15:17:03 GMT", "last-modified": "Fri, 30 Oct 2026 15:17:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "29753ebd2a3ce94fcc404b98f969d788355a72dc588134d2f6bcbea28c3aeb46", "previousChainHash": "d7ca539e8a4c74b7685577423eabcba5f991b00537dfac9bd6f2ace19a6081e6" }, { "id": "41e8980a-67c4-4f64-b8be-860c3f88a3ec", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-29T15:17:19.243Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Thu, 29 Oct 2026 15:17:19 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "41e8980a-67c4-4f64-b8be-860c3f88a3ec", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e901993bfbe4b09ab2b7dbed2e802d6ffe98a0e0a5a3e744d61c0d735ad18a1a", "previousChainHash": "ef4800ab8b570df098051915f2539018acacc8cb590b7d9661fbbeb800d0923a" }, { "id": "526a5d82-4e4d-4518-a10e-8a9e702c3576", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-29T15:17:16.106Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Thu, 29 Oct 2026 15:17:16 GMT", "last-modified": "Thu, 29 Oct 2026 15:08:19 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d7ca539e8a4c74b7685577423eabcba5f991b00537dfac9bd6f2ace19a6081e6", "previousChainHash": "059adf482f2c6705db0c2c27bba76d7238665804adb332becc924a48fa25842f" }, { "id": "18195552-e1ec-4031-838e-29b1ca308b47", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-28T15:17:09.709Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Wed, 28 Oct 2026 15:17:09 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "18195552-e1ec-4031-838e-29b1ca308b47", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ef4800ab8b570df098051915f2539018acacc8cb590b7d9661fbbeb800d0923a", "previousChainHash": "6f90f9a8b9ea0937c092791017517d0d046390b4e2bd92cc240fb3e41d4aaa87" }, { "id": "09b360b4-3d75-4add-9d09-f3a6c7837a00", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-28T15:17:04.903Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Wed, 28 Oct 2026 15:17:04 GMT", "last-modified": "Wed, 28 Oct 2026 15:09:59 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "059adf482f2c6705db0c2c27bba76d7238665804adb332becc924a48fa25842f", "previousChainHash": "10fd12caee3b65fe9812cf2810a3d325e17383933fbd9be01de94326bfba708a" }, { "id": "4e511da3-d0b9-45a5-8863-252b0d69ea06", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-27T15:17:07.560Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Tue, 27 Oct 2026 15:17:07 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "4e511da3-d0b9-45a5-8863-252b0d69ea06", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6f90f9a8b9ea0937c092791017517d0d046390b4e2bd92cc240fb3e41d4aaa87", "previousChainHash": "826be96a12b26b4c94780e3ab3e51d653edc310cd27493af22331dac19a3ee8f" }, { "id": "19dd5297-5013-4f65-95b7-501bcc7a0800", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-27T15:17:04.339Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Tue, 27 Oct 2026 15:17:04 GMT", "last-modified": "Tue, 27 Oct 2026 15:11:50 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "10fd12caee3b65fe9812cf2810a3d325e17383933fbd9be01de94326bfba708a", "previousChainHash": "bf36fe6e2b410ad900206f52225e937d37de699798d684904fbf4dc09c7712cf" }, { "id": "c4c0df84-22ac-41e0-b759-00d76ff2676d", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-26T15:17:06.503Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Mon, 26 Oct 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "c4c0df84-22ac-41e0-b759-00d76ff2676d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "826be96a12b26b4c94780e3ab3e51d653edc310cd27493af22331dac19a3ee8f", "previousChainHash": "34337eeaa1b8c6e92db84f4636368c33f06cae0e8011cfd8d21c306489b65f34" }, { "id": "d2b1138b-93f4-4488-b4c3-52aa3a6476d8", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-26T15:17:03.114Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Mon, 26 Oct 2026 15:17:03 GMT", "last-modified": "Mon, 26 Oct 2026 15:12:04 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bf36fe6e2b410ad900206f52225e937d37de699798d684904fbf4dc09c7712cf", "previousChainHash": "f554acef75266708bfd45a716946c11e3702b25c520c54c9e62387f65cad63ac" }, { "id": "e92199f7-8466-48cd-a01a-bca3db4f0d56", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-25T15:17:06.573Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sun, 25 Oct 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "e92199f7-8466-48cd-a01a-bca3db4f0d56", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "34337eeaa1b8c6e92db84f4636368c33f06cae0e8011cfd8d21c306489b65f34", "previousChainHash": "dc45d4ad994ded73437ea8ca2e363aa58b381af962ac6850be51046c1182f955" }, { "id": "076b0310-1ead-4330-84f5-bc28804c8988", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-25T15:17:03.515Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sun, 25 Oct 2026 15:17:03 GMT", "last-modified": "Sun, 25 Oct 2026 15:10:23 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f554acef75266708bfd45a716946c11e3702b25c520c54c9e62387f65cad63ac", "previousChainHash": "7880c9bd20e1d362592c0cca361566e32324124f0a35a64b0e2146d3e42d6949" }, { "id": "50fb46c2-812a-47c5-b714-4ff9f3c287b2", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-24T15:17:04.828Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sat, 24 Oct 2026 15:17:04 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "50fb46c2-812a-47c5-b714-4ff9f3c287b2", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "dc45d4ad994ded73437ea8ca2e363aa58b381af962ac6850be51046c1182f955", "previousChainHash": "71cf4c28332b5577d97e88a81f276cd4404806ece146beb417fbe8a4521872ca" }, { "id": "ad7b536a-7160-4784-9381-f35e74128f72", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-24T15:17:01.792Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sat, 24 Oct 2026 15:17:01 GMT", "last-modified": "Sat, 24 Oct 2026 15:00:20 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7880c9bd20e1d362592c0cca361566e32324124f0a35a64b0e2146d3e42d6949", "previousChainHash": "ef612bcf1271e49903ed92e2f95cf42dd73cbffb0d5a5bfc04727b841d0c3c2d" }, { "id": "0cdc68ac-0152-4bf3-95bc-f2abd76b59d1", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-23T15:17:06.870Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Fri, 23 Oct 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "0cdc68ac-0152-4bf3-95bc-f2abd76b59d1", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "71cf4c28332b5577d97e88a81f276cd4404806ece146beb417fbe8a4521872ca", "previousChainHash": "d0401ba240717a0b72d2cdc54a8522fbb485df2cc622d804e1c971e3891fc97f" }, { "id": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-23T15:17:02.816Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Fri, 23 Oct 2026 15:17:02 GMT", "last-modified": "Fri, 23 Oct 2026 15:16:11 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "cfc5a777ebf2a44330f6a07e6a16e41ab94e0d1c05a732012f08cf6f86451d6d", "artifactBytes": 1378, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 71, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/scripts/CustomNews.js" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "ed345dcd-28ab-49c0-a3e3-26d59a1ace83", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ef612bcf1271e49903ed92e2f95cf42dd73cbffb0d5a5bfc04727b841d0c3c2d", "previousChainHash": "ce47188a1a16f4194e7de84ac9faa70c637574e30269c4d36a79439dfb03a57d" }, { "id": "fd923666-bc65-487c-ae7d-398d8b841053", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-22T15:17:12.964Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Thu, 22 Oct 2026 15:17:12 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "fd923666-bc65-487c-ae7d-398d8b841053", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d0401ba240717a0b72d2cdc54a8522fbb485df2cc622d804e1c971e3891fc97f", "previousChainHash": "f00755fd5d17f11ac26286f6533f8c19b5b178ca53716bb032814ef68b8e73f2" }, { "id": "fd733323-7d2f-4780-b7dd-0e563fdd36b0", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-22T15:17:09.309Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Thu, 22 Oct 2026 15:17:09 GMT", "last-modified": "Thu, 22 Oct 2026 15:17:09 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "fad8a37c40bf85e344b17ae483d9d81e363a90eae8c4f53ea2978787e6ef63da", "artifactBytes": 1326, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 32, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 38, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 44, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 47, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 53, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 56, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 32, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 68, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 65, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 35, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "fd733323-7d2f-4780-b7dd-0e563fdd36b0", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ce47188a1a16f4194e7de84ac9faa70c637574e30269c4d36a79439dfb03a57d", "previousChainHash": "f015bc8951dd146570f37c2df9f930d7cbd52dc1216bce005e6172b85ac1e119" }, { "id": "15e48a31-88b8-4d86-a0a2-e3c5f94f59f8", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-21T15:17:07.662Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Wed, 21 Oct 2026 15:17:07 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "15e48a31-88b8-4d86-a0a2-e3c5f94f59f8", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f00755fd5d17f11ac26286f6533f8c19b5b178ca53716bb032814ef68b8e73f2", "previousChainHash": "70490a92e9cb26b1583fe9bc3d94db68e4076f15e2825c946d39cc11346adf44" }, { "id": "afbfbea5-d734-4109-9232-ea4250470e57", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-21T15:17:04.420Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Wed, 21 Oct 2026 15:17:04 GMT", "last-modified": "Wed, 21 Oct 2026 15:17:04 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "9eebdaf40c8b8a0fad6b0a80e97286ccbdcc66927c75648cc6fb22c12d4191ea", "artifactBytes": 1348, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b0c29a54-6654-4b86-aca6-d51abcbee93c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f015bc8951dd146570f37c2df9f930d7cbd52dc1216bce005e6172b85ac1e119", "previousChainHash": "d34d053e774afac80b287f5437567bf71373181fd47b660d14be24fde7deb245" }, { "id": "86abdce5-bdf5-4cad-ae76-c33c2aff5eb4", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-20T15:17:05.725Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Tue, 20 Oct 2026 15:17:05 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "86abdce5-bdf5-4cad-ae76-c33c2aff5eb4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "70490a92e9cb26b1583fe9bc3d94db68e4076f15e2825c946d39cc11346adf44", "previousChainHash": "1db5378a360eb1e9f717e5f91352a1ba663fbaa9fb8374c4079e84c13fec7a2f" }, { "id": "3a4fc779-b4e9-47d4-844d-7b2cdab9f080", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-20T15:17:02.792Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Tue, 20 Oct 2026 15:17:02 GMT", "last-modified": "Tue, 20 Oct 2026 15:14:10 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "9eebdaf40c8b8a0fad6b0a80e97286ccbdcc66927c75648cc6fb22c12d4191ea", "artifactBytes": 1348, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b0c29a54-6654-4b86-aca6-d51abcbee93c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d34d053e774afac80b287f5437567bf71373181fd47b660d14be24fde7deb245", "previousChainHash": "761fdf5a0b5c6461701264a137b1d1ba21998db0ae47a6fe36a4df4381ed0347" }, { "id": "5c158fd6-b93e-4773-9d6d-89c78bc0f0a8", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-19T15:17:10.514Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Mon, 19 Oct 2026 15:17:10 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "5c158fd6-b93e-4773-9d6d-89c78bc0f0a8", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1db5378a360eb1e9f717e5f91352a1ba663fbaa9fb8374c4079e84c13fec7a2f", "previousChainHash": "7e6b1899d8eba9c6063e86ea2a0a519fdb726d984f51a260fdeb472c00e4632d" }, { "id": "b18a24b0-ab41-4fcd-81c5-ee4d6f88313e", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-19T15:17:07.106Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Mon, 19 Oct 2026 15:17:07 GMT", "last-modified": "Mon, 19 Oct 2026 15:09:48 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "9eebdaf40c8b8a0fad6b0a80e97286ccbdcc66927c75648cc6fb22c12d4191ea", "artifactBytes": 1348, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b0c29a54-6654-4b86-aca6-d51abcbee93c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "761fdf5a0b5c6461701264a137b1d1ba21998db0ae47a6fe36a4df4381ed0347", "previousChainHash": "9d44667da3ec786321af9e0bb6d79e5f3eb88516affbbd0d20fa008e83a8644e" }, { "id": "d00ff6d2-f0e9-4b46-93a0-032e14c30318", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-10-18T15:17:06.089Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sun, 18 Oct 2026 15:17:06 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "d00ff6d2-f0e9-4b46-93a0-032e14c30318", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7e6b1899d8eba9c6063e86ea2a0a519fdb726d984f51a260fdeb472c00e4632d", "previousChainHash": "f2c5899e5b24af0b4a732cea8756f1ae5755b9c075923ebef4d774b119d057e7" }, { "id": "f9cf8a36-67c4-43cb-abe7-3fcce2a2578e", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-10-18T15:17:02.976Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sun, 18 Oct 2026 15:17:02 GMT", "last-modified": "Sun, 18 Oct 2026 15:17:02 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "9eebdaf40c8b8a0fad6b0a80e97286ccbdcc66927c75648cc6fb22c12d4191ea", "artifactBytes": 1348, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0+agents-2026-09-20", "registryVersion": "2026-09-20", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/author-sitemap-content.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 69, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ], "state": "disallowed" }, "OAI-AdsBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 3, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 4, "pattern": "/zephr*" }, { "directive": "disallow", "line": 5, "pattern": "/blaize*" }, { "directive": "disallow", "line": 6, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 7, "pattern": "/*.rss" }, { "directive": "disallow", "line": 8, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 10, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 11, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 12, "pattern": "/gallery/" }, { "directive": "disallow", "line": 13, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 14, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 15, "pattern": "/apdata/" }, { "directive": "disallow", "line": 16, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 17, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 18, "pattern": "/press-release/*" }, { "directive": "disallow", "line": 19, "pattern": "/*/widgets/" }, { "directive": "disallow", "line": 20, "pattern": "/*/widget/" }, { "directive": "disallow", "line": 21, "pattern": "/*/scripts/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "b0c29a54-6654-4b86-aca6-d51abcbee93c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9d44667da3ec786321af9e0bb6d79e5f3eb88516affbbd0d20fa008e83a8644e", "previousChainHash": "dd9181c9a8c0532e9dc50c239d262e67f5d039989b58f493d6c27bb70bc80077" } ] }