{ "domain": "usatoday.com", "recordsNotConclusions": true, "events": [ { "id": "4076e692-869d-45b2-8b35-3a5caab97f73", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-12T19:17:15.960Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Wed, 12 Aug 2026 19:17:15 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "4076e692-869d-45b2-8b35-3a5caab97f73", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "47f576e350263844d6782bdaccc5a6b257c989e1aa03039e686d265fea17f6fb", "previousChainHash": "7f3b359699c9f863c7860c9b4b3af1510542f51c42ad012b8701c807e05b38bc" }, { "id": "acbe1793-0ad2-4348-8f12-cc3d8828cc4d", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-12T19:17:13.079Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Wed, 12 Aug 2026 19:17:13 GMT", "etag": "W/\"6a7cabd3-943\"", "last-modified": "Wed, 12 Aug 2026 17:22:27 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "9bf8ba4edf177c3d6d293262c25e2149df11a62620e7a66e1938801526dd301c", "artifactBytes": 14456, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 40, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 43, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 49, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" }, { "directive": "allow", "line": 53, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 55, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 56, "pattern": "/shopping/" }, { "directive": "allow", "line": 57, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 58, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 61, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 64, "pattern": "/" }, { "directive": "allow", "line": 65, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 66, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 67, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 68, "pattern": "/shopping/" }, { "directive": "allow", "line": 69, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 70, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" }, { "directive": "allow", "line": 110, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 111, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 112, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 113, "pattern": "/shopping/" }, { "directive": "allow", "line": 114, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 115, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 130, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 133, "pattern": "/" }, { "directive": "allow", "line": 134, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 135, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 136, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 137, "pattern": "/shopping/" }, { "directive": "allow", "line": 138, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 139, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 142, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 145, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 148, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 151, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 154, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 157, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 160, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 163, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 166, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 169, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 172, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 175, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 178, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 181, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 184, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 187, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 190, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 193, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 196, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 199, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 202, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 205, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 208, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 211, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 214, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 217, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 220, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 223, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 226, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 229, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 232, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 235, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 238, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 241, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 244, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 247, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 250, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 253, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 256, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 259, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 262, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 265, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 268, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 271, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 274, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 277, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 280, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 283, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 286, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 289, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 292, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 295, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 298, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 301, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 304, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 307, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 310, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 313, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 316, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 319, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 322, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 325, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 328, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 331, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 334, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 337, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 340, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 343, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 346, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 349, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 352, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 355, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 358, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 361, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 364, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 367, "pattern": "/" }, { "directive": "allow", "line": 368, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 369, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 370, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 371, "pattern": "/shopping/" }, { "directive": "allow", "line": 372, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 373, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 898, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 901, "pattern": "/errors" }, { "directive": "disallow", "line": 902, "pattern": "/interactive/" }, { "directive": "disallow", "line": 903, "pattern": "/userauth/" }, { "directive": "disallow", "line": 904, "pattern": "/ugc/" }, { "directive": "disallow", "line": 905, "pattern": "/feeds/" }, { "directive": "disallow", "line": 906, "pattern": "/services/" }, { "directive": "disallow", "line": 907, "pattern": "/facebook/" }, { "directive": "disallow", "line": 908, "pattern": "/version-info/" }, { "directive": "disallow", "line": 909, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 910, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 911, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 912, "pattern": "/search" }, { "directive": "disallow", "line": 913, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 914, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 915, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 916, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 917, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 918, "pattern": "/optimus" }, { "directive": "disallow", "line": 919, "pattern": "/ux-train" }, { "directive": "disallow", "line": 920, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 921, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 922, "pattern": "/pbd/" }, { "directive": "disallow", "line": 923, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 925, "pattern": "/dcc/" }, { "directive": "disallow", "line": 926, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 927, "pattern": "/dc/" }, { "directive": "disallow", "line": 928, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 929, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 930, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 931, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 932, "pattern": "/zxc/" }, { "directive": "disallow", "line": 933, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 934, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 935, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 936, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 937, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 938, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 939, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 940, "pattern": "/yourtake" }, { "directive": "disallow", "line": 941, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 942, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 943, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 944, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 945, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 946, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 947, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 950, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 61, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 40, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 148, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 145, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 64, "pattern": "/" }, { "directive": "allow", "line": 65, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 66, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 67, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 68, "pattern": "/shopping/" }, { "directive": "allow", "line": 69, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 70, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" }, { "directive": "allow", "line": 53, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 55, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 56, "pattern": "/shopping/" }, { "directive": "allow", "line": 57, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 58, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 133, "pattern": "/" }, { "directive": "allow", "line": 134, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 135, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 136, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 137, "pattern": "/shopping/" }, { "directive": "allow", "line": 138, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 139, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" }, { "directive": "allow", "line": 110, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 111, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 112, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 113, "pattern": "/shopping/" }, { "directive": "allow", "line": 114, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 115, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "2838e2f3-4189-44a4-b88f-7240cdee4174", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "983e2149315d65db69a5e0e064b3f9a367506b1a4c67bc9d469e03a7e71b5dda", "previousChainHash": "153694dc3ef498b3ba3e13c6864d5a8f1d0709aee61c743948e2562e573670cd" }, { "id": "2e0b72b3-e140-417f-aa86-974a823eb941", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-11T19:17:16.413Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Tue, 11 Aug 2026 19:17:16 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "2e0b72b3-e140-417f-aa86-974a823eb941", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7f3b359699c9f863c7860c9b4b3af1510542f51c42ad012b8701c807e05b38bc", "previousChainHash": "b0958c0fb486b1d91823fefd556b963cb66c73bfdce38a471ec09aded77e5b56" }, { "id": "2838e2f3-4189-44a4-b88f-7240cdee4174", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-11T19:17:13.059Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Tue, 11 Aug 2026 19:17:13 GMT", "etag": "W/\"6a7b5d4b-943\"", "last-modified": "Tue, 11 Aug 2026 17:35:07 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "9bf8ba4edf177c3d6d293262c25e2149df11a62620e7a66e1938801526dd301c", "artifactBytes": 14456, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 40, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 43, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 49, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" }, { "directive": "allow", "line": 53, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 55, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 56, "pattern": "/shopping/" }, { "directive": "allow", "line": 57, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 58, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 61, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 64, "pattern": "/" }, { "directive": "allow", "line": 65, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 66, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 67, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 68, "pattern": "/shopping/" }, { "directive": "allow", "line": 69, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 70, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" }, { "directive": "allow", "line": 110, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 111, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 112, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 113, "pattern": "/shopping/" }, { "directive": "allow", "line": 114, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 115, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 130, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 133, "pattern": "/" }, { "directive": "allow", "line": 134, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 135, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 136, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 137, "pattern": "/shopping/" }, { "directive": "allow", "line": 138, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 139, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 142, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 145, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 148, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 151, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 154, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 157, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 160, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 163, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 166, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 169, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 172, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 175, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 178, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 181, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 184, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 187, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 190, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 193, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 196, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 199, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 202, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 205, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 208, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 211, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 214, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 217, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 220, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 223, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 226, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 229, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 232, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 235, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 238, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 241, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 244, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 247, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 250, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 253, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 256, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 259, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 262, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 265, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 268, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 271, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 274, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 277, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 280, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 283, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 286, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 289, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 292, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 295, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 298, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 301, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 304, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 307, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 310, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 313, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 316, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 319, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 322, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 325, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 328, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 331, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 334, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 337, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 340, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 343, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 346, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 349, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 352, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 355, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 358, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 361, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 364, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 367, "pattern": "/" }, { "directive": "allow", "line": 368, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 369, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 370, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 371, "pattern": "/shopping/" }, { "directive": "allow", "line": 372, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 373, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 898, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 901, "pattern": "/errors" }, { "directive": "disallow", "line": 902, "pattern": "/interactive/" }, { "directive": "disallow", "line": 903, "pattern": "/userauth/" }, { "directive": "disallow", "line": 904, "pattern": "/ugc/" }, { "directive": "disallow", "line": 905, "pattern": "/feeds/" }, { "directive": "disallow", "line": 906, "pattern": "/services/" }, { "directive": "disallow", "line": 907, "pattern": "/facebook/" }, { "directive": "disallow", "line": 908, "pattern": "/version-info/" }, { "directive": "disallow", "line": 909, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 910, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 911, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 912, "pattern": "/search" }, { "directive": "disallow", "line": 913, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 914, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 915, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 916, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 917, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 918, "pattern": "/optimus" }, { "directive": "disallow", "line": 919, "pattern": "/ux-train" }, { "directive": "disallow", "line": 920, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 921, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 922, "pattern": "/pbd/" }, { "directive": "disallow", "line": 923, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 925, "pattern": "/dcc/" }, { "directive": "disallow", "line": 926, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 927, "pattern": "/dc/" }, { "directive": "disallow", "line": 928, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 929, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 930, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 931, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 932, "pattern": "/zxc/" }, { "directive": "disallow", "line": 933, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 934, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 935, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 936, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 937, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 938, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 939, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 940, "pattern": "/yourtake" }, { "directive": "disallow", "line": 941, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 942, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 943, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 944, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 945, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 946, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 947, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 950, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 61, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 40, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 148, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 145, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 64, "pattern": "/" }, { "directive": "allow", "line": 65, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 66, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 67, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 68, "pattern": "/shopping/" }, { "directive": "allow", "line": 69, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 70, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" }, { "directive": "allow", "line": 53, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 55, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 56, "pattern": "/shopping/" }, { "directive": "allow", "line": 57, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 58, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 133, "pattern": "/" }, { "directive": "allow", "line": 134, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 135, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 136, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 137, "pattern": "/shopping/" }, { "directive": "allow", "line": 138, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 139, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" }, { "directive": "allow", "line": 110, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 111, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 112, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 113, "pattern": "/shopping/" }, { "directive": "allow", "line": 114, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 115, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "2838e2f3-4189-44a4-b88f-7240cdee4174", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "153694dc3ef498b3ba3e13c6864d5a8f1d0709aee61c743948e2562e573670cd", "previousChainHash": "034c48274d85f8f52a5d447fa14e4b0164d7b340457933f067a7b31f057d0372" }, { "id": "c1053977-a4c3-4d5c-a7d0-7c30920df566", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-10T19:17:16.416Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Mon, 10 Aug 2026 19:17:16 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "c1053977-a4c3-4d5c-a7d0-7c30920df566", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "b0958c0fb486b1d91823fefd556b963cb66c73bfdce38a471ec09aded77e5b56", "previousChainHash": "3f28b0aa1795b21499b4c2e7b4cb89ebec6b478135d2b4174d5580c4b890343b" }, { "id": "bca40552-b435-413b-a191-b2d737aa8411", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-10T19:17:13.656Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Mon, 10 Aug 2026 19:17:13 GMT", "etag": "W/\"6a79f9be-93e\"", "last-modified": "Mon, 10 Aug 2026 16:18:06 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b0d291be591427b6e82afed04fbb27e6aa8a7aced7ca196ae36cd7894e5afa98", "artifactBytes": 14312, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 114, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 117, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 123, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 126, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 137, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 146, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 149, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 152, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 155, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 158, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 161, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 164, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 167, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 170, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 173, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 176, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 179, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 182, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 185, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 188, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 191, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 194, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 197, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 200, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 203, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 206, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 209, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 212, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 215, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 218, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 221, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 224, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 227, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 230, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 233, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 236, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 239, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 242, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 245, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 248, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 251, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 254, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 257, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 260, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 263, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 266, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 269, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 272, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 275, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 278, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 281, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 284, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 287, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 290, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 293, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 296, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 299, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 302, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 305, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 308, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 311, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 314, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 317, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 320, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 323, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 326, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 329, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 332, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 335, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 338, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 341, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 344, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 347, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 350, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 353, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 356, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 359, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 362, "pattern": "/" }, { "directive": "allow", "line": 363, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 364, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 365, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 366, "pattern": "/shopping/" }, { "directive": "allow", "line": 367, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/errors" }, { "directive": "disallow", "line": 896, "pattern": "/interactive/" }, { "directive": "disallow", "line": 897, "pattern": "/userauth/" }, { "directive": "disallow", "line": 898, "pattern": "/ugc/" }, { "directive": "disallow", "line": 899, "pattern": "/feeds/" }, { "directive": "disallow", "line": 900, "pattern": "/services/" }, { "directive": "disallow", "line": 901, "pattern": "/facebook/" }, { "directive": "disallow", "line": 902, "pattern": "/version-info/" }, { "directive": "disallow", "line": 903, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 904, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 905, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 906, "pattern": "/search" }, { "directive": "disallow", "line": 907, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 908, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 909, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 910, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 911, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 912, "pattern": "/optimus" }, { "directive": "disallow", "line": 913, "pattern": "/ux-train" }, { "directive": "disallow", "line": 914, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 915, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 916, "pattern": "/pbd/" }, { "directive": "disallow", "line": 917, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 919, "pattern": "/dcc/" }, { "directive": "disallow", "line": 920, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 921, "pattern": "/dc/" }, { "directive": "disallow", "line": 922, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 923, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 925, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 926, "pattern": "/zxc/" }, { "directive": "disallow", "line": 927, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 928, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 929, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 930, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 931, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 932, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 933, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 934, "pattern": "/yourtake" }, { "directive": "disallow", "line": 935, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 936, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 937, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 938, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 939, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 940, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 941, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 944, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "12b66fb1-0381-4f30-8a52-844a6d5d9e91", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "034c48274d85f8f52a5d447fa14e4b0164d7b340457933f067a7b31f057d0372", "previousChainHash": "977f78cfec2a69272f3245cc468c52484c49e83705dd2581e942432da8404076" }, { "id": "9fb3e38f-4654-4d4b-8207-7d5dd58e1648", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-09T19:17:15.823Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Sun, 09 Aug 2026 19:17:15 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "9fb3e38f-4654-4d4b-8207-7d5dd58e1648", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3f28b0aa1795b21499b4c2e7b4cb89ebec6b478135d2b4174d5580c4b890343b", "previousChainHash": "1b6cb6d0759279b2ed58d82c093e36344312f6741d00ff195c944304b6a40417" }, { "id": "9e6f0cb5-224e-45b4-ac26-62df09c74c91", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-09T19:17:13.157Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Sun, 09 Aug 2026 19:17:13 GMT", "etag": "W/\"6a75f5d5-93e\"", "last-modified": "Fri, 07 Aug 2026 15:12:21 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b0d291be591427b6e82afed04fbb27e6aa8a7aced7ca196ae36cd7894e5afa98", "artifactBytes": 14312, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 114, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 117, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 123, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 126, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 137, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 146, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 149, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 152, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 155, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 158, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 161, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 164, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 167, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 170, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 173, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 176, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 179, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 182, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 185, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 188, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 191, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 194, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 197, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 200, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 203, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 206, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 209, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 212, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 215, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 218, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 221, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 224, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 227, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 230, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 233, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 236, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 239, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 242, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 245, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 248, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 251, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 254, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 257, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 260, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 263, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 266, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 269, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 272, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 275, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 278, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 281, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 284, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 287, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 290, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 293, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 296, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 299, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 302, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 305, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 308, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 311, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 314, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 317, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 320, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 323, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 326, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 329, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 332, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 335, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 338, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 341, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 344, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 347, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 350, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 353, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 356, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 359, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 362, "pattern": "/" }, { "directive": "allow", "line": 363, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 364, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 365, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 366, "pattern": "/shopping/" }, { "directive": "allow", "line": 367, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/errors" }, { "directive": "disallow", "line": 896, "pattern": "/interactive/" }, { "directive": "disallow", "line": 897, "pattern": "/userauth/" }, { "directive": "disallow", "line": 898, "pattern": "/ugc/" }, { "directive": "disallow", "line": 899, "pattern": "/feeds/" }, { "directive": "disallow", "line": 900, "pattern": "/services/" }, { "directive": "disallow", "line": 901, "pattern": "/facebook/" }, { "directive": "disallow", "line": 902, "pattern": "/version-info/" }, { "directive": "disallow", "line": 903, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 904, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 905, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 906, "pattern": "/search" }, { "directive": "disallow", "line": 907, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 908, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 909, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 910, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 911, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 912, "pattern": "/optimus" }, { "directive": "disallow", "line": 913, "pattern": "/ux-train" }, { "directive": "disallow", "line": 914, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 915, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 916, "pattern": "/pbd/" }, { "directive": "disallow", "line": 917, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 919, "pattern": "/dcc/" }, { "directive": "disallow", "line": 920, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 921, "pattern": "/dc/" }, { "directive": "disallow", "line": 922, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 923, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 925, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 926, "pattern": "/zxc/" }, { "directive": "disallow", "line": 927, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 928, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 929, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 930, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 931, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 932, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 933, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 934, "pattern": "/yourtake" }, { "directive": "disallow", "line": 935, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 936, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 937, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 938, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 939, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 940, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 941, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 944, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "12b66fb1-0381-4f30-8a52-844a6d5d9e91", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "977f78cfec2a69272f3245cc468c52484c49e83705dd2581e942432da8404076", "previousChainHash": "95c3f593aab2249ec5e45492af2040c715ab226fa68bce07674f709523325f67" }, { "id": "c262e7b4-ef70-4e7a-90b1-06dedbad6f58", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-08T19:17:16.614Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Sat, 08 Aug 2026 19:17:16 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "c262e7b4-ef70-4e7a-90b1-06dedbad6f58", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1b6cb6d0759279b2ed58d82c093e36344312f6741d00ff195c944304b6a40417", "previousChainHash": "2a8db8170068293603d53240b68059770951ce58ba09d4907c2c763650c588e9" }, { "id": "d1b6ffc8-2486-468f-bbea-acdbefc16154", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-08T19:17:13.834Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Sat, 08 Aug 2026 19:17:13 GMT", "etag": "W/\"6a75f5d5-93e\"", "last-modified": "Fri, 07 Aug 2026 15:12:21 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "b0d291be591427b6e82afed04fbb27e6aa8a7aced7ca196ae36cd7894e5afa98", "artifactBytes": 14312, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 114, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 117, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 123, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 126, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 137, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 146, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 149, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 152, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 155, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 158, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 161, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 164, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 167, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 170, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 173, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 176, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 179, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 182, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 185, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 188, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 191, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 194, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 197, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 200, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 203, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 206, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 209, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 212, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 215, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 218, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 221, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 224, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 227, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 230, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 233, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 236, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 239, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 242, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 245, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 248, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 251, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 254, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 257, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 260, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 263, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 266, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 269, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 272, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 275, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 278, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 281, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 284, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 287, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 290, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 293, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 296, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 299, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 302, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 305, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 308, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 311, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 314, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 317, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 320, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 323, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 326, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 329, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 332, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 335, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 338, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 341, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 344, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 347, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 350, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 353, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 356, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 359, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 362, "pattern": "/" }, { "directive": "allow", "line": 363, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 364, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 365, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 366, "pattern": "/shopping/" }, { "directive": "allow", "line": 367, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/errors" }, { "directive": "disallow", "line": 896, "pattern": "/interactive/" }, { "directive": "disallow", "line": 897, "pattern": "/userauth/" }, { "directive": "disallow", "line": 898, "pattern": "/ugc/" }, { "directive": "disallow", "line": 899, "pattern": "/feeds/" }, { "directive": "disallow", "line": 900, "pattern": "/services/" }, { "directive": "disallow", "line": 901, "pattern": "/facebook/" }, { "directive": "disallow", "line": 902, "pattern": "/version-info/" }, { "directive": "disallow", "line": 903, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 904, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 905, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 906, "pattern": "/search" }, { "directive": "disallow", "line": 907, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 908, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 909, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 910, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 911, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 912, "pattern": "/optimus" }, { "directive": "disallow", "line": 913, "pattern": "/ux-train" }, { "directive": "disallow", "line": 914, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 915, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 916, "pattern": "/pbd/" }, { "directive": "disallow", "line": 917, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 919, "pattern": "/dcc/" }, { "directive": "disallow", "line": 920, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 921, "pattern": "/dc/" }, { "directive": "disallow", "line": 922, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 923, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 925, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 926, "pattern": "/zxc/" }, { "directive": "disallow", "line": 927, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 928, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 929, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 930, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 931, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 932, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 933, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 934, "pattern": "/yourtake" }, { "directive": "disallow", "line": 935, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 936, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 937, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 938, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 939, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 940, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 941, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 944, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "12b66fb1-0381-4f30-8a52-844a6d5d9e91", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "95c3f593aab2249ec5e45492af2040c715ab226fa68bce07674f709523325f67", "previousChainHash": "64037eccfa97110625669906110f192d5945cca7a7d377b14a2d1ce87e3a939f" }, { "id": "23911364-ed06-4e16-9e36-bef22960c138", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-07T19:17:16.317Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Fri, 07 Aug 2026 19:17:16 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "23911364-ed06-4e16-9e36-bef22960c138", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "2a8db8170068293603d53240b68059770951ce58ba09d4907c2c763650c588e9", "previousChainHash": "1860789b094ad3c2b92c6ed9522a6828269aed75877d552f80a3f97ce22b24f5" }, { "id": "12b66fb1-0381-4f30-8a52-844a6d5d9e91", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-07T19:17:13.062Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Fri, 07 Aug 2026 19:17:13 GMT", "etag": "W/\"6a75f5d5-93e\"", "last-modified": "Fri, 07 Aug 2026 15:12:21 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "b0d291be591427b6e82afed04fbb27e6aa8a7aced7ca196ae36cd7894e5afa98", "artifactBytes": 14312, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 42, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 114, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 117, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 123, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 126, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 137, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 146, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 149, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 152, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 155, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 158, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 161, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 164, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 167, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 170, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 173, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 176, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 179, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 182, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 185, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 188, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 191, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 194, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 197, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 200, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 203, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 206, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 209, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 212, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 215, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 218, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 221, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 224, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 227, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 230, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 233, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 236, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 239, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 242, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 245, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 248, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 251, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 254, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 257, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 260, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 263, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 266, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 269, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 272, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 275, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 278, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 281, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 284, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 287, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 290, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 293, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 296, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 299, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 302, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 305, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 308, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 311, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 314, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 317, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 320, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 323, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 326, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 329, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 332, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 335, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 338, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 341, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 344, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 347, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 350, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 353, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 356, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 359, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 362, "pattern": "/" }, { "directive": "allow", "line": 363, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 364, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 365, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 366, "pattern": "/shopping/" }, { "directive": "allow", "line": 367, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/errors" }, { "directive": "disallow", "line": 896, "pattern": "/interactive/" }, { "directive": "disallow", "line": 897, "pattern": "/userauth/" }, { "directive": "disallow", "line": 898, "pattern": "/ugc/" }, { "directive": "disallow", "line": 899, "pattern": "/feeds/" }, { "directive": "disallow", "line": 900, "pattern": "/services/" }, { "directive": "disallow", "line": 901, "pattern": "/facebook/" }, { "directive": "disallow", "line": 902, "pattern": "/version-info/" }, { "directive": "disallow", "line": 903, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 904, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 905, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 906, "pattern": "/search" }, { "directive": "disallow", "line": 907, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 908, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 909, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 910, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 911, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 912, "pattern": "/optimus" }, { "directive": "disallow", "line": 913, "pattern": "/ux-train" }, { "directive": "disallow", "line": 914, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 915, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 916, "pattern": "/pbd/" }, { "directive": "disallow", "line": 917, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 919, "pattern": "/dcc/" }, { "directive": "disallow", "line": 920, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 921, "pattern": "/dc/" }, { "directive": "disallow", "line": 922, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 923, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 925, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 926, "pattern": "/zxc/" }, { "directive": "disallow", "line": 927, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 928, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 929, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 930, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 931, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 932, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 933, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 934, "pattern": "/yourtake" }, { "directive": "disallow", "line": 935, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 936, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 937, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 938, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 939, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 940, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 941, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 944, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 120, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 59, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 143, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 140, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 62, "pattern": "/" }, { "directive": "allow", "line": 63, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 64, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 65, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 66, "pattern": "/shopping/" }, { "directive": "allow", "line": 67, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 51, "pattern": "/" }, { "directive": "allow", "line": 52, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 55, "pattern": "/shopping/" }, { "directive": "allow", "line": 56, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 129, "pattern": "/" }, { "directive": "allow", "line": 130, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 131, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 132, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 133, "pattern": "/shopping/" }, { "directive": "allow", "line": 134, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" }, { "directive": "allow", "line": 107, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 108, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 109, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 110, "pattern": "/shopping/" }, { "directive": "allow", "line": 111, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "12b66fb1-0381-4f30-8a52-844a6d5d9e91", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "64037eccfa97110625669906110f192d5945cca7a7d377b14a2d1ce87e3a939f", "previousChainHash": "74e423999a55315145f2639df1e0c9fa2a3967f3295878159a2b62ca619bc97a" }, { "id": "f26d1301-f31c-4e34-8b31-2e55cf5c0d2b", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-06T19:17:16.243Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Thu, 06 Aug 2026 19:17:16 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "f26d1301-f31c-4e34-8b31-2e55cf5c0d2b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "1860789b094ad3c2b92c6ed9522a6828269aed75877d552f80a3f97ce22b24f5", "previousChainHash": "85a09786413c00494325ad8f3b2925c3e488f42b5f6a97cd7d9327dc12a70c81" }, { "id": "e2da2a0a-565b-4d33-b9d5-5f7ff5db7208", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-06T19:17:13.537Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Thu, 06 Aug 2026 19:17:13 GMT", "etag": "W/\"6a723edf-8ff\"", "last-modified": "Tue, 04 Aug 2026 19:34:55 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "4fb71131ab42f6ed3c1250088b8832541c5984a3e5e473238c20be30698a504d", "artifactBytes": 13502, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 42, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" }, { "directive": "allow", "line": 51, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 52, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 54, "pattern": "/shopping/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" }, { "directive": "allow", "line": 61, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 62, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 63, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 64, "pattern": "/shopping/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" }, { "directive": "allow", "line": 104, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 105, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 106, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 107, "pattern": "/shopping/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 110, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 113, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 116, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 119, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 122, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 125, "pattern": "/" }, { "directive": "allow", "line": 126, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 127, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 128, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 129, "pattern": "/shopping/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 132, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 135, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 138, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 141, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 144, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 147, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 150, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 153, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 156, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 159, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 162, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 165, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 168, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 171, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 174, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 177, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 180, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 183, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 186, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 189, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 192, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 195, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 198, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 201, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 204, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 207, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 210, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 213, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 216, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 219, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 222, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 225, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 228, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 231, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 234, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 237, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 240, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 243, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 246, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 249, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 252, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 255, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 258, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 261, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 264, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 267, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 270, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 273, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 276, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 279, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 282, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 285, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 288, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 291, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 294, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 297, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 300, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 303, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 306, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 309, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 312, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 315, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 318, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 321, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 324, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 327, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 330, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 333, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 336, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 339, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 342, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 345, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 348, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 351, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 354, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 357, "pattern": "/" }, { "directive": "allow", "line": 358, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 359, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 360, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 361, "pattern": "/shopping/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 364, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 367, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/errors" }, { "directive": "disallow", "line": 890, "pattern": "/interactive/" }, { "directive": "disallow", "line": 891, "pattern": "/userauth/" }, { "directive": "disallow", "line": 892, "pattern": "/ugc/" }, { "directive": "disallow", "line": 893, "pattern": "/feeds/" }, { "directive": "disallow", "line": 894, "pattern": "/services/" }, { "directive": "disallow", "line": 895, "pattern": "/facebook/" }, { "directive": "disallow", "line": 896, "pattern": "/version-info/" }, { "directive": "disallow", "line": 897, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 898, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 899, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 900, "pattern": "/search" }, { "directive": "disallow", "line": 901, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 902, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 903, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 904, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 905, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 906, "pattern": "/optimus" }, { "directive": "disallow", "line": 907, "pattern": "/ux-train" }, { "directive": "disallow", "line": 908, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 909, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 910, "pattern": "/pbd/" }, { "directive": "disallow", "line": 911, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 912, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 913, "pattern": "/dcc/" }, { "directive": "disallow", "line": 914, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 915, "pattern": "/dc/" }, { "directive": "disallow", "line": 916, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 917, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 919, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 920, "pattern": "/zxc/" }, { "directive": "disallow", "line": 921, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 922, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 923, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 924, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 925, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 926, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 927, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 928, "pattern": "/yourtake" }, { "directive": "disallow", "line": 929, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 930, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 931, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 932, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 933, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 934, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 935, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 938, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 116, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 138, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 135, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" }, { "directive": "allow", "line": 61, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 62, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 63, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 64, "pattern": "/shopping/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" }, { "directive": "allow", "line": 51, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 52, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 54, "pattern": "/shopping/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 125, "pattern": "/" }, { "directive": "allow", "line": 126, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 127, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 128, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 129, "pattern": "/shopping/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" }, { "directive": "allow", "line": 104, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 105, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 106, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 107, "pattern": "/shopping/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "fb1695a6-1760-4d8b-bc5e-d3ae81fd8d81", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "74e423999a55315145f2639df1e0c9fa2a3967f3295878159a2b62ca619bc97a", "previousChainHash": "88844f57f76ed8d3ec59fcf5612d2f7e32c0632023774d98ead5c91655cf3159" }, { "id": "3138587e-65fb-451e-af63-b425e9994872", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-05T19:17:17.102Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Wed, 05 Aug 2026 19:17:17 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "3138587e-65fb-451e-af63-b425e9994872", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "85a09786413c00494325ad8f3b2925c3e488f42b5f6a97cd7d9327dc12a70c81", "previousChainHash": "afa67809244e5fa27fa1c2a6c58af2a78e1a60805cd26d1f8a142374c04d943e" }, { "id": "ee0fdd3c-4191-4ed4-91ae-bfa27ef76d7c", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-05T19:17:14.170Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Wed, 05 Aug 2026 19:17:14 GMT", "etag": "W/\"6a723edf-8ff\"", "last-modified": "Tue, 04 Aug 2026 19:34:55 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "4fb71131ab42f6ed3c1250088b8832541c5984a3e5e473238c20be30698a504d", "artifactBytes": 13502, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 42, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" }, { "directive": "allow", "line": 51, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 52, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 54, "pattern": "/shopping/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" }, { "directive": "allow", "line": 61, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 62, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 63, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 64, "pattern": "/shopping/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" }, { "directive": "allow", "line": 104, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 105, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 106, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 107, "pattern": "/shopping/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 110, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 113, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 116, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 119, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 122, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 125, "pattern": "/" }, { "directive": "allow", "line": 126, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 127, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 128, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 129, "pattern": "/shopping/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 132, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 135, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 138, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 141, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 144, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 147, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 150, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 153, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 156, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 159, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 162, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 165, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 168, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 171, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 174, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 177, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 180, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 183, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 186, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 189, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 192, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 195, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 198, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 201, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 204, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 207, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 210, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 213, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 216, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 219, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 222, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 225, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 228, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 231, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 234, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 237, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 240, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 243, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 246, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 249, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 252, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 255, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 258, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 261, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 264, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 267, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 270, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 273, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 276, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 279, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 282, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 285, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 288, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 291, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 294, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 297, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 300, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 303, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 306, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 309, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 312, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 315, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 318, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 321, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 324, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 327, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 330, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 333, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 336, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 339, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 342, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 345, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 348, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 351, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 354, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 357, "pattern": "/" }, { "directive": "allow", "line": 358, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 359, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 360, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 361, "pattern": "/shopping/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 364, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 367, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/errors" }, { "directive": "disallow", "line": 890, "pattern": "/interactive/" }, { "directive": "disallow", "line": 891, "pattern": "/userauth/" }, { "directive": "disallow", "line": 892, "pattern": "/ugc/" }, { "directive": "disallow", "line": 893, "pattern": "/feeds/" }, { "directive": "disallow", "line": 894, "pattern": "/services/" }, { "directive": "disallow", "line": 895, "pattern": "/facebook/" }, { "directive": "disallow", "line": 896, "pattern": "/version-info/" }, { "directive": "disallow", "line": 897, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 898, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 899, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 900, "pattern": "/search" }, { "directive": "disallow", "line": 901, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 902, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 903, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 904, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 905, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 906, "pattern": "/optimus" }, { "directive": "disallow", "line": 907, "pattern": "/ux-train" }, { "directive": "disallow", "line": 908, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 909, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 910, "pattern": "/pbd/" }, { "directive": "disallow", "line": 911, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 912, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 913, "pattern": "/dcc/" }, { "directive": "disallow", "line": 914, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 915, "pattern": "/dc/" }, { "directive": "disallow", "line": 916, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 917, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 919, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 920, "pattern": "/zxc/" }, { "directive": "disallow", "line": 921, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 922, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 923, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 924, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 925, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 926, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 927, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 928, "pattern": "/yourtake" }, { "directive": "disallow", "line": 929, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 930, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 931, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 932, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 933, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 934, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 935, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 938, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 116, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 138, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 135, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" }, { "directive": "allow", "line": 61, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 62, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 63, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 64, "pattern": "/shopping/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" }, { "directive": "allow", "line": 51, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 52, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 54, "pattern": "/shopping/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 125, "pattern": "/" }, { "directive": "allow", "line": 126, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 127, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 128, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 129, "pattern": "/shopping/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" }, { "directive": "allow", "line": 104, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 105, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 106, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 107, "pattern": "/shopping/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "fb1695a6-1760-4d8b-bc5e-d3ae81fd8d81", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "88844f57f76ed8d3ec59fcf5612d2f7e32c0632023774d98ead5c91655cf3159", "previousChainHash": "adf4a6051ce22aded6ba3ee038afab2452390c457bc993bced1afbf8d6f19d6f" }, { "id": "13197345-53e5-420f-937d-efaae4bc9dac", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-04T19:17:22.927Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Tue, 04 Aug 2026 19:17:22 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "13197345-53e5-420f-937d-efaae4bc9dac", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "afa67809244e5fa27fa1c2a6c58af2a78e1a60805cd26d1f8a142374c04d943e", "previousChainHash": null }, { "id": "fb1695a6-1760-4d8b-bc5e-d3ae81fd8d81", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-04T19:17:14.907Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Tue, 04 Aug 2026 19:17:14 GMT", "etag": "W/\"6a673cc8-8ff\"", "last-modified": "Mon, 27 Jul 2026 11:11:04 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "4fb71131ab42f6ed3c1250088b8832541c5984a3e5e473238c20be30698a504d", "artifactBytes": 13502, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 41, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 42, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 43, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" }, { "directive": "allow", "line": 51, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 52, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 54, "pattern": "/shopping/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" }, { "directive": "allow", "line": 61, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 62, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 63, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 64, "pattern": "/shopping/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 67, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 70, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" }, { "directive": "allow", "line": 104, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 105, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 106, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 107, "pattern": "/shopping/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 110, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 113, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 116, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 119, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 122, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 125, "pattern": "/" }, { "directive": "allow", "line": 126, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 127, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 128, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 129, "pattern": "/shopping/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 132, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 135, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 138, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 141, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 144, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 147, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 150, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 153, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 156, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 159, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 162, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 165, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 168, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 171, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 174, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 177, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 180, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 183, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 186, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 189, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 192, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 195, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 198, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 201, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 204, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 207, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 210, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 213, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 216, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 219, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 222, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 225, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 228, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 231, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 234, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 237, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 240, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 243, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 246, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 249, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 252, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 255, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 258, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 261, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 264, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 267, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 270, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 273, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 276, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 279, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 282, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 285, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 288, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 291, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 294, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 297, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 300, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 303, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 306, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 309, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 312, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 315, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 318, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 321, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 324, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 327, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 330, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 333, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 336, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 339, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 342, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 345, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 348, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 351, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 354, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 357, "pattern": "/" }, { "directive": "allow", "line": 358, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 359, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 360, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 361, "pattern": "/shopping/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 364, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 367, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 370, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 373, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/errors" }, { "directive": "disallow", "line": 890, "pattern": "/interactive/" }, { "directive": "disallow", "line": 891, "pattern": "/userauth/" }, { "directive": "disallow", "line": 892, "pattern": "/ugc/" }, { "directive": "disallow", "line": 893, "pattern": "/feeds/" }, { "directive": "disallow", "line": 894, "pattern": "/services/" }, { "directive": "disallow", "line": 895, "pattern": "/facebook/" }, { "directive": "disallow", "line": 896, "pattern": "/version-info/" }, { "directive": "disallow", "line": 897, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 898, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 899, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 900, "pattern": "/search" }, { "directive": "disallow", "line": 901, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 902, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 903, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 904, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 905, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 906, "pattern": "/optimus" }, { "directive": "disallow", "line": 907, "pattern": "/ux-train" }, { "directive": "disallow", "line": 908, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 909, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 910, "pattern": "/pbd/" }, { "directive": "disallow", "line": 911, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 912, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 913, "pattern": "/dcc/" }, { "directive": "disallow", "line": 914, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 915, "pattern": "/dc/" }, { "directive": "disallow", "line": 916, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 917, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 918, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 919, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 920, "pattern": "/zxc/" }, { "directive": "disallow", "line": 921, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 922, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 923, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 924, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 925, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 926, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 927, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 928, "pattern": "/yourtake" }, { "directive": "disallow", "line": 929, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 930, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 931, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 932, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 933, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 934, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 935, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 938, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 116, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 57, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 138, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 135, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 60, "pattern": "/" }, { "directive": "allow", "line": 61, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 62, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 63, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 64, "pattern": "/shopping/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 50, "pattern": "/" }, { "directive": "allow", "line": 51, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 52, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 53, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 54, "pattern": "/shopping/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 125, "pattern": "/" }, { "directive": "allow", "line": 126, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 127, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 128, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 129, "pattern": "/shopping/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" }, { "directive": "allow", "line": 104, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 105, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 106, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 107, "pattern": "/shopping/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "fb1695a6-1760-4d8b-bc5e-d3ae81fd8d81", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "adf4a6051ce22aded6ba3ee038afab2452390c457bc993bced1afbf8d6f19d6f", "previousChainHash": null } ] }