{ "domain": "apnews.com", "recordsNotConclusions": true, "events": [ { "id": "890c9a56-43b6-49d3-a055-c0f2183e9841", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-13T15:17:15.979Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Thu, 13 Aug 2026 15:17:15 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "890c9a56-43b6-49d3-a055-c0f2183e9841", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6708c44d3a4cdd1c7b446a95e2ffc851fc6579af5ec7628c7533eb14b486212b", "previousChainHash": "40121ebe1919109cf3495b50973fb1bab5b7a38d6fda48ce66e6189ce167f161" }, { "id": "8ff56722-9e16-45ad-a7da-520a8b544b6e", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-13T15:17:13.151Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Thu, 13 Aug 2026 15:17:13 GMT", "last-modified": "Thu, 13 Aug 2026 15:11:57 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "contentSignals": [], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-content-signals-0.2.0", "registryVersion": "2026-08-13", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "51b0b7d1c78b948d5f6229fa140cd3b3502d051b137fb957fcc368ddc242317e", "previousChainHash": "1f019f0091b1f5cdcf735f495cb799e183e5e15e1efd1ea13edc17c42ed8bc07" }, { "id": "8b43fa00-e652-4d9f-b889-5a7df5614d3b", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-12T15:17:16.765Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Wed, 12 Aug 2026 15:17:16 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "8b43fa00-e652-4d9f-b889-5a7df5614d3b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "40121ebe1919109cf3495b50973fb1bab5b7a38d6fda48ce66e6189ce167f161", "previousChainHash": "e1bf69a839c31cf84c1764769f16619b3426df1fff1d0ce8df7ad0b71f1dd6b0" }, { "id": "765c83ff-6468-4635-b2fc-056adfeaccb9", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-12T15:17:13.694Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Wed, 12 Aug 2026 15:17:13 GMT", "last-modified": "Wed, 12 Aug 2026 15:12:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1f019f0091b1f5cdcf735f495cb799e183e5e15e1efd1ea13edc17c42ed8bc07", "previousChainHash": "753e4e81a8c59dca96c719be8ba876ded49625a7c594c385e699b1735d486a51" }, { "id": "537ea60b-5db2-4a95-bfa1-caaa47fcc2b7", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-11T15:17:16.233Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Tue, 11 Aug 2026 15:17:16 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "537ea60b-5db2-4a95-bfa1-caaa47fcc2b7", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e1bf69a839c31cf84c1764769f16619b3426df1fff1d0ce8df7ad0b71f1dd6b0", "previousChainHash": "ea73dc68750313d8a342a79c881c27c9ce2b0c0751df158a6cf3ca613fb0048c" }, { "id": "ac99a1ca-c522-4166-809b-0243bd86f832", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-11T15:17:13.141Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Tue, 11 Aug 2026 15:17:13 GMT", "last-modified": "Tue, 11 Aug 2026 15:12:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "753e4e81a8c59dca96c719be8ba876ded49625a7c594c385e699b1735d486a51", "previousChainHash": "f30cdab134a1099824c5d16f7ad483bec6fd13334772de9971cc93c694a945da" }, { "id": "6a8b1949-d4e4-48c3-8154-d99a959d8193", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-10T15:17:24.254Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Mon, 10 Aug 2026 15:17:24 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "6a8b1949-d4e4-48c3-8154-d99a959d8193", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ea73dc68750313d8a342a79c881c27c9ce2b0c0751df158a6cf3ca613fb0048c", "previousChainHash": "a4fd61aad3230e87798a5adfa4e3e00ae671194d094e60d7b0330781b3970e60" }, { "id": "ff11289c-ac21-42d9-a40f-0e6ef3697d5f", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-10T15:17:16.018Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Mon, 10 Aug 2026 15:17:16 GMT", "last-modified": "Mon, 10 Aug 2026 15:16:15 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f30cdab134a1099824c5d16f7ad483bec6fd13334772de9971cc93c694a945da", "previousChainHash": "02cf3da688ed46131e9d9c92af7b85569ea2999845ace44330f7ac40abd8d018" }, { "id": "fd17cece-4d24-4b3c-9fce-722cf95f0bac", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-09T15:17:15.503Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sun, 09 Aug 2026 15:17:15 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "fd17cece-4d24-4b3c-9fce-722cf95f0bac", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a4fd61aad3230e87798a5adfa4e3e00ae671194d094e60d7b0330781b3970e60", "previousChainHash": "e09bbee9ddffe3aff2072c8a58491b39f99a1f1cc2b5df2f8ac2c20fb7b0ffb3" }, { "id": "abe7b43d-4640-4b2e-8569-04e6c40b4162", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-09T15:17:12.865Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sun, 09 Aug 2026 15:17:12 GMT", "last-modified": "Sun, 09 Aug 2026 14:56:57 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "02cf3da688ed46131e9d9c92af7b85569ea2999845ace44330f7ac40abd8d018", "previousChainHash": "1a084ba5d3b739990959cb3b42b2b67ee1afa29376f34197268b4f0d860d3289" }, { "id": "9a11da62-fb8a-4f75-afc2-caa69f3a32df", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-08T15:17:16.319Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Sat, 08 Aug 2026 15:17:16 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "9a11da62-fb8a-4f75-afc2-caa69f3a32df", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e09bbee9ddffe3aff2072c8a58491b39f99a1f1cc2b5df2f8ac2c20fb7b0ffb3", "previousChainHash": "e75aa4453a846e1cf19a24100e815155d12075b90cf2f70eb034bf3f5e0a3547" }, { "id": "ec6c4a3b-dfea-47b8-89fe-40867d502caf", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-08T15:17:13.249Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Sat, 08 Aug 2026 15:17:13 GMT", "last-modified": "Sat, 08 Aug 2026 14:42:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1a084ba5d3b739990959cb3b42b2b67ee1afa29376f34197268b4f0d860d3289", "previousChainHash": "c56de7c7fd2e7227cea180e0de4dbec827b8c3934e2d0347ce5ecf64bd1cb79b" }, { "id": "eee22ffd-0f08-4160-a632-1ca0d8721d52", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-07T15:17:23.150Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Fri, 07 Aug 2026 15:17:23 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "eee22ffd-0f08-4160-a632-1ca0d8721d52", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e75aa4453a846e1cf19a24100e815155d12075b90cf2f70eb034bf3f5e0a3547", "previousChainHash": "e0d201530a5889cc8ac35673e528f11595ddf7ba411ad2e1296bb126f634ae65" }, { "id": "2d996bd2-21eb-40ea-9684-ea9323d5b515", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-07T15:17:14.923Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Fri, 07 Aug 2026 15:17:14 GMT", "last-modified": "Fri, 07 Aug 2026 15:12:03 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c56de7c7fd2e7227cea180e0de4dbec827b8c3934e2d0347ce5ecf64bd1cb79b", "previousChainHash": "d5ef50bf7581a96003f55656ee375a4d6c935aefce557d2399c6e9cfd54260b9" }, { "id": "59784762-3992-4990-8806-30c56788a6e8", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-06T15:17:16.427Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Thu, 06 Aug 2026 15:17:16 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "59784762-3992-4990-8806-30c56788a6e8", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e0d201530a5889cc8ac35673e528f11595ddf7ba411ad2e1296bb126f634ae65", "previousChainHash": "665aa62505a66dded434bec2ba4803bdcf64b23d5211922c70c57a64b26feb7d" }, { "id": "439061cb-3437-48b7-9399-5aff732498ef", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-06T15:17:13.420Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Thu, 06 Aug 2026 15:17:13 GMT", "last-modified": "Thu, 06 Aug 2026 15:15:15 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d5ef50bf7581a96003f55656ee375a4d6c935aefce557d2399c6e9cfd54260b9", "previousChainHash": "56f663306d086e59a99f0d994c2341cb2676ad216c301dac8caba665a38e5d49" }, { "id": "e404ce14-597b-471e-844c-76393e8c2fb8", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-05T15:17:16.832Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Wed, 05 Aug 2026 15:17:16 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "e404ce14-597b-471e-844c-76393e8c2fb8", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "665aa62505a66dded434bec2ba4803bdcf64b23d5211922c70c57a64b26feb7d", "previousChainHash": "a8ccd6a6d9151491bdc7f6d559aa20696909fb8dfd65ba3a72b1295cffcdb169" }, { "id": "39f6242e-ae3e-4553-a623-0754d00b5122", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-05T15:17:13.766Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Wed, 05 Aug 2026 15:17:13 GMT", "last-modified": "Wed, 05 Aug 2026 15:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "56f663306d086e59a99f0d994c2341cb2676ad216c301dac8caba665a38e5d49", "previousChainHash": "af31ea2ce0c630b735c5cb22cbcce27eb4dbd40182aaef5a17a406483ff083b4" }, { "id": "1c895fab-2429-4d68-9fd0-f62fcadfdcc4", "domain": "apnews.com", "signal": "llms", "requestedUrl": "https://apnews.com/llms.txt", "finalUrl": "https://apnews.com/llms.txt", "fetchedAt": "2026-08-04T15:17:33.883Z", "httpStatus": 404, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/html;charset=UTF-8", "date": "Tue, 04 Aug 2026 15:17:31 GMT", "server": "cloudflare" }, "parseStatus": "too_large", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "too_large", "canonicalEventId": "1c895fab-2429-4d68-9fd0-f62fcadfdcc4", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a8ccd6a6d9151491bdc7f6d559aa20696909fb8dfd65ba3a72b1295cffcdb169", "previousChainHash": null }, { "id": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "domain": "apnews.com", "signal": "robots", "requestedUrl": "https://apnews.com/robots.txt", "finalUrl": "https://apnews.com/robots.txt", "fetchedAt": "2026-08-04T15:17:22.564Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=30, s-maxage=31536005", "content-type": "text/plain", "date": "Tue, 04 Aug 2026 15:17:22 GMT", "last-modified": "Tue, 04 Aug 2026 15:17:22 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "b5e768463fa30256aea2ecbda68a69bcc59b3556a1cd00e84a1793bbf22a3e88", "artifactBytes": 1238, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ] }, { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 36, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 45, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ] }, { "agents": [ "Amazonbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ] }, { "agents": [ "Bytedance" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://apnews.com/ap-sitemap.xml", "https://apnews.com/news-sitemap-content.xml", "https://apnews.com/hubs-sitemap-content.xml", "https://apnews.com/video-sitemap.xml", "https://apnews.com/projects/elections-2026/sitemap-index.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 54, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 66, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 63, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 39, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 33, "pattern": "/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 3, "pattern": "*_ptid=*" }, { "directive": "disallow", "line": 4, "pattern": "*?prx_t=*" }, { "directive": "disallow", "line": 5, "pattern": "/zephr*" }, { "directive": "disallow", "line": 6, "pattern": "/blaize*" }, { "directive": "disallow", "line": 7, "pattern": "/search?q=*" }, { "directive": "disallow", "line": 8, "pattern": "/*.rss" }, { "directive": "disallow", "line": 9, "pattern": "/buyline-shopping/*" }, { "directive": "disallow", "line": 10, "pattern": "/buyline-personal-finance/*" }, { "directive": "disallow", "line": 11, "pattern": "*?jw_start" }, { "directive": "disallow", "line": 12, "pattern": "*&jw_start" }, { "directive": "disallow", "line": 13, "pattern": "/gallery/" }, { "directive": "disallow", "line": 14, "pattern": "/config/newrelic/prod" }, { "directive": "disallow", "line": 15, "pattern": "/parsely-api-endpoint" }, { "directive": "disallow", "line": 16, "pattern": "/apdata/" }, { "directive": "disallow", "line": 17, "pattern": "/article/border-immigration-election-2182fec11fa008ec96c4202e5656d19e" }, { "directive": "disallow", "line": 18, "pattern": "/api/v2/feed/" }, { "directive": "disallow", "line": 19, "pattern": "/press-release/*" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 48, "pattern": "/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "3af061b0-f3b4-4b57-bdba-d10ea9afd18b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "af31ea2ce0c630b735c5cb22cbcce27eb4dbd40182aaef5a17a406483ff083b4", "previousChainHash": null } ] }