{ "domain": "usatoday.com", "recordsNotConclusions": true, "observations": [ { "id": "4076e692-869d-45b2-8b35-3a5caab97f73", "domain": "usatoday.com", "signal": "llms", "requestedUrl": "https://usatoday.com/llms.txt", "finalUrl": "https://usatoday.com/llms.txt", "fetchedAt": "2026-08-12T19:17:15.960Z", "httpStatus": 301, "responseHeaders": { "cache-control": "max-age=300", "content-length": "0", "date": "Wed, 12 Aug 2026 19:17:15 GMT", "location": "https://www.usatoday.com/llms.txt", "retry-after": "0", "server": "cloudflare" }, "parseStatus": "http_error", "changed": false, "artifactSha256": null, "artifactBytes": null, "parserVersion": null, "parsed": null, "errorCode": "redirect_cross_host", "canonicalEventId": "4076e692-869d-45b2-8b35-3a5caab97f73", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "47f576e350263844d6782bdaccc5a6b257c989e1aa03039e686d265fea17f6fb", "previousChainHash": "7f3b359699c9f863c7860c9b4b3af1510542f51c42ad012b8701c807e05b38bc" }, { "id": "acbe1793-0ad2-4348-8f12-cc3d8828cc4d", "domain": "usatoday.com", "signal": "robots", "requestedUrl": "https://usatoday.com/robots.txt", "finalUrl": "https://www.usatoday.com/robots.txt", "fetchedAt": "2026-08-12T19:17:13.079Z", "httpStatus": 200, "responseHeaders": { "content-type": "text/plain", "date": "Wed, 12 Aug 2026 19:17:13 GMT", "etag": "W/\"6a7cabd3-943\"", "last-modified": "Wed, 12 Aug 2026 17:22:27 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "9bf8ba4edf177c3d6d293262c25e2149df11a62620e7a66e1938801526dd301c", "artifactBytes": 14456, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "anthropic-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 4, "pattern": "/" } ] }, { "agents": [ "AmazonBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 7, "pattern": "/" } ] }, { "agents": [ "Claude" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 10, "pattern": "/" } ] }, { "agents": [ "Claude-Web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 13, "pattern": "/" } ] }, { "agents": [ "claudeweb" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 16, "pattern": "/" } ] }, { "agents": [ "Amzn-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 19, "pattern": "/" } ] }, { "agents": [ "Amzn-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 22, "pattern": "/" } ] }, { "agents": [ "AndiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 25, "pattern": "/" } ] }, { "agents": [ "cohere-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 28, "pattern": "/" } ] }, { "agents": [ "ClaudeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ] }, { "agents": [ "ChatGPT-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 40, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Googlebot-News" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 43, "pattern": "/story/sponsor-story/" }, { "directive": "disallow", "line": 44, "pattern": "/picture-gallery/sponsor-story/" }, { "directive": "disallow", "line": 45, "pattern": "/videos/sponsor-story/" }, { "directive": "disallow", "line": 46, "pattern": "/longform/sponsor-story/" }, { "directive": "disallow", "line": 47, "pattern": "/pages/interactives/sponsor-story/" }, { "directive": "disallow", "line": 48, "pattern": "/interactives/sponsor-story/" }, { "directive": "disallow", "line": 49, "pattern": "/videos/embed/" } ] }, { "agents": [ "GPTBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" }, { "directive": "allow", "line": 53, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 55, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 56, "pattern": "/shopping/" }, { "directive": "allow", "line": 57, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 58, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "CCBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 61, "pattern": "/" } ] }, { "agents": [ "Google-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 64, "pattern": "/" }, { "directive": "allow", "line": 65, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 66, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 67, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 68, "pattern": "/shopping/" }, { "directive": "allow", "line": 69, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 70, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "DataForSeoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 73, "pattern": "/" } ] }, { "agents": [ "Diffbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 76, "pattern": "/" } ] }, { "agents": [ "Diffbot-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 79, "pattern": "/" } ] }, { "agents": [ "Kagibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 82, "pattern": "/" } ] }, { "agents": [ "FacebookBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 85, "pattern": "/" } ] }, { "agents": [ "magpie-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 88, "pattern": "/" } ] }, { "agents": [ "NewsNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 91, "pattern": "/" } ] }, { "agents": [ "news-please" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 94, "pattern": "/" } ] }, { "agents": [ "omgili" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 97, "pattern": "/" } ] }, { "agents": [ "omgilibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 100, "pattern": "/" } ] }, { "agents": [ "peer39_crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 103, "pattern": "/" } ] }, { "agents": [ "peer39_crawler/1.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 106, "pattern": "/" } ] }, { "agents": [ "PerplexityBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" }, { "directive": "allow", "line": 110, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 111, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 112, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 113, "pattern": "/shopping/" }, { "directive": "allow", "line": 114, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 115, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Scrapy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 118, "pattern": "/" } ] }, { "agents": [ "TurnitinBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 121, "pattern": "/" } ] }, { "agents": [ "Applebot-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 127, "pattern": "/" } ] }, { "agents": [ "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 130, "pattern": "/" } ] }, { "agents": [ "OAI-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 133, "pattern": "/" }, { "directive": "allow", "line": 134, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 135, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 136, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 137, "pattern": "/shopping/" }, { "directive": "allow", "line": 138, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 139, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "Bytespider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 142, "pattern": "/" } ] }, { "agents": [ "Claude-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 145, "pattern": "/" } ] }, { "agents": [ "Claude-SearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 148, "pattern": "/" } ] }, { "agents": [ "AI2Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 151, "pattern": "/" } ] }, { "agents": [ "AhrefsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 154, "pattern": "/" } ] }, { "agents": [ "Ai2Bot-Dolma" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 157, "pattern": "/" } ] }, { "agents": [ "Applebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 160, "pattern": "/" } ] }, { "agents": [ "Arquivo-web-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 163, "pattern": "/" } ] }, { "agents": [ "AspiegelBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 166, "pattern": "/" } ] }, { "agents": [ "AwarioRssBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 169, "pattern": "/" } ] }, { "agents": [ "AwarioSmartBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 172, "pattern": "/" } ] }, { "agents": [ "Barkrowler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 175, "pattern": "/" } ] }, { "agents": [ "coccocbot-web" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 178, "pattern": "/" } ] }, { "agents": [ "cohere-training-data-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 181, "pattern": "/" } ] }, { "agents": [ "crawler4j" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 184, "pattern": "/" } ] }, { "agents": [ "Crawlspace" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 187, "pattern": "/" } ] }, { "agents": [ "dotbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 190, "pattern": "/" } ] }, { "agents": [ "DuckAssistBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 193, "pattern": "/" } ] }, { "agents": [ "FriendlyCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 196, "pattern": "/" } ] }, { "agents": [ "Google-CloudVertexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 199, "pattern": "/" } ] }, { "agents": [ "https://hada.news" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 202, "pattern": "/" } ] }, { "agents": [ "https://www.imediaethics.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 205, "pattern": "/" } ] }, { "agents": [ "iaskspider/2.0" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 208, "pattern": "/" } ] }, { "agents": [ "ICC-Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 211, "pattern": "/" } ] }, { "agents": [ "icc-crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 214, "pattern": "/" } ] }, { "agents": [ "ICC Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 217, "pattern": "/" } ] }, { "agents": [ "ImagesiftBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 220, "pattern": "/" } ] }, { "agents": [ "img2dataset" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 223, "pattern": "/" } ] }, { "agents": [ "ISSCyberRiskCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 226, "pattern": "/" } ] }, { "agents": [ "JenkersBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 229, "pattern": "/" } ] }, { "agents": [ "Kangaroo Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 232, "pattern": "/" } ] }, { "agents": [ "LivelapBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 235, "pattern": "/" } ] }, { "agents": [ "MauiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 238, "pattern": "/" } ] }, { "agents": [ "Meltwater" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 241, "pattern": "/" } ] }, { "agents": [ "Meta-ExternalFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 244, "pattern": "/" } ] }, { "agents": [ "MJ12bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 247, "pattern": "/" } ] }, { "agents": [ "Mojeek" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 250, "pattern": "/" } ] }, { "agents": [ "MojeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 253, "pattern": "/" } ] }, { "agents": [ "MoodleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 256, "pattern": "/" } ] }, { "agents": [ "NeticleBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 259, "pattern": "/" } ] }, { "agents": [ "Netvibes" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 262, "pattern": "/" } ] }, { "agents": [ "PanguBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 265, "pattern": "/" } ] }, { "agents": [ "PetalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 268, "pattern": "/" } ] }, { "agents": [ "SeekportBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 271, "pattern": "/" } ] }, { "agents": [ "Seekr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 274, "pattern": "/" } ] }, { "agents": [ "SemrushBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 277, "pattern": "/" } ] }, { "agents": [ "SeznamBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 280, "pattern": "/" } ] }, { "agents": [ "Sidetrade Indexer Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 283, "pattern": "/" } ] }, { "agents": [ "Timpibot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 286, "pattern": "/" } ] }, { "agents": [ "Turnitin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 289, "pattern": "/" } ] }, { "agents": [ "VelenPublicWebCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 292, "pattern": "/" } ] }, { "agents": [ "Webzio-Extended" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 295, "pattern": "/" } ] }, { "agents": [ "yacy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 298, "pattern": "/" } ] }, { "agents": [ "yacybot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 301, "pattern": "/" } ] }, { "agents": [ "Yandex" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 304, "pattern": "/" } ] }, { "agents": [ "YandexAdditional" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 307, "pattern": "/" } ] }, { "agents": [ "YandexAdditionalBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 310, "pattern": "/" } ] }, { "agents": [ "YandexBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 313, "pattern": "/" } ] }, { "agents": [ "YandexImages" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 316, "pattern": "/" } ] }, { "agents": [ "YandexNews" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 319, "pattern": "/" } ] }, { "agents": [ "Baiduspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 322, "pattern": "/" } ] }, { "agents": [ "Baiduspider-image" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 325, "pattern": "/" } ] }, { "agents": [ "BaiduSpider-render" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 328, "pattern": "/" } ] }, { "agents": [ "Baiduspider-video" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 331, "pattern": "/" } ] }, { "agents": [ "Sogou Web Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 334, "pattern": "/" } ] }, { "agents": [ "yisouspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 337, "pattern": "/" } ] }, { "agents": [ "YouBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 340, "pattern": "/" } ] }, { "agents": [ "ia_archiver" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 343, "pattern": "/" } ] }, { "agents": [ "archive.org_bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 346, "pattern": "/" } ] }, { "agents": [ "ia_archiver-web.archive.org" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 349, "pattern": "/" } ] }, { "agents": [ "aiHitBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 352, "pattern": "/" } ] }, { "agents": [ "Amazon Kendra" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 355, "pattern": "/" } ] }, { "agents": [ "Big Sur AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 358, "pattern": "/" } ] }, { "agents": [ "Brandwatch" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 361, "pattern": "/" } ] }, { "agents": [ "Bravest" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 364, "pattern": "/" } ] }, { "agents": [ "ChatGPT Operator" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 367, "pattern": "/" }, { "directive": "allow", "line": 368, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 369, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 370, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 371, "pattern": "/shopping/" }, { "directive": "allow", "line": 372, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 373, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ] }, { "agents": [ "ChatGPT-PageFetcher" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 376, "pattern": "/" } ] }, { "agents": [ "OpenAI-CheckBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 379, "pattern": "/" } ] }, { "agents": [ "OpenAI-Explore" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 382, "pattern": "/" } ] }, { "agents": [ "OpenAI Feedback" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 385, "pattern": "/" } ] }, { "agents": [ "OpenAI-SnippetBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 388, "pattern": "/" } ] }, { "agents": [ "Cotoyogi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 391, "pattern": "/" } ] }, { "agents": [ "DigitalOceanGenAICrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 394, "pattern": "/" } ] }, { "agents": [ "Echobot Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 397, "pattern": "/" } ] }, { "agents": [ "EchoboxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 400, "pattern": "/" } ] }, { "agents": [ "Factset_spyderbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 403, "pattern": "/" } ] }, { "agents": [ "Grok" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 406, "pattern": "/" } ] }, { "agents": [ "iaskspider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 409, "pattern": "/" } ] }, { "agents": [ "imgproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 412, "pattern": "/" } ] }, { "agents": [ "LINER Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 415, "pattern": "/" } ] }, { "agents": [ "MistralAI-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 418, "pattern": "/" } ] }, { "agents": [ "netEstate Imprint Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 421, "pattern": "/" } ] }, { "agents": [ "NovaAct" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 424, "pattern": "/" } ] }, { "agents": [ "QualifiedBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 427, "pattern": "/" } ] }, { "agents": [ "SBIntuitionsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 430, "pattern": "/" } ] }, { "agents": [ "sbintuitionsbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 433, "pattern": "/" } ] }, { "agents": [ "SemrushBot-OCOB" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 436, "pattern": "/" } ] }, { "agents": [ "SemrushBotSwa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 439, "pattern": "/" } ] }, { "agents": [ "Sidetrade" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 442, "pattern": "/" } ] }, { "agents": [ "TikTokSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 445, "pattern": "/" } ] }, { "agents": [ "WARDBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 448, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot V1" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 451, "pattern": "/" } ] }, { "agents": [ "TaraGroup Intelligent Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 454, "pattern": "/" } ] }, { "agents": [ "DeepSeekBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 457, "pattern": "/" } ] }, { "agents": [ "ChatGLM-Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 460, "pattern": "/" } ] }, { "agents": [ "GoogleOther" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 463, "pattern": "/" } ] }, { "agents": [ "Google AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 466, "pattern": "/" } ] }, { "agents": [ "Google-Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 469, "pattern": "/" } ] }, { "agents": [ "shapbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 472, "pattern": "/" } ] }, { "agents": [ "FirecrawlAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 475, "pattern": "/" } ] }, { "agents": [ "Crawl4AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 478, "pattern": "/" } ] }, { "agents": [ "imageSpider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 481, "pattern": "/" } ] }, { "agents": [ "laion-huggingface-processor" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 484, "pattern": "/" } ] }, { "agents": [ "LCC" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 487, "pattern": "/" } ] }, { "agents": [ "Spider" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 490, "pattern": "/" } ] }, { "agents": [ "Datenbank Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 493, "pattern": "/" } ] }, { "agents": [ "Cloudflare-AutoRAG" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 496, "pattern": "/" } ] }, { "agents": [ "CloudflareBrowserRenderingCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 499, "pattern": "/" } ] }, { "agents": [ "meta-webindexer" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 502, "pattern": "/" } ] }, { "agents": [ "Bravebot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 505, "pattern": "/" } ] }, { "agents": [ "AddSearchBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 508, "pattern": "/" } ] }, { "agents": [ "Anomura" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 511, "pattern": "/" } ] }, { "agents": [ "atlassian-bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 514, "pattern": "/" } ] }, { "agents": [ "Channel3Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 517, "pattern": "/" } ] }, { "agents": [ "LinkupBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 520, "pattern": "/" } ] }, { "agents": [ "ZanistaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 523, "pattern": "/" } ] }, { "agents": [ "Gemini-Deep-Research" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 526, "pattern": "/" } ] }, { "agents": [ "Google-NotebookLM" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 529, "pattern": "/" } ] }, { "agents": [ "Perplexity-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ] }, { "agents": [ "Perplexity Search Bot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 535, "pattern": "/" } ] }, { "agents": [ "Poetbot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 538, "pattern": "/" } ] }, { "agents": [ "Poseidon Research Crawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 541, "pattern": "/" } ] }, { "agents": [ "ProRataInc" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 544, "pattern": "/" } ] }, { "agents": [ "QuillBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 547, "pattern": "/" } ] }, { "agents": [ "Tiingo News Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 550, "pattern": "/" } ] }, { "agents": [ "iboubot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 553, "pattern": "/" } ] }, { "agents": [ "TavilyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 556, "pattern": "/" } ] }, { "agents": [ "NeevaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 559, "pattern": "/" } ] }, { "agents": [ "PhindBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 562, "pattern": "/" } ] }, { "agents": [ "AI2Bot-DeepResearchEval" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 565, "pattern": "/" } ] }, { "agents": [ "KlaviyoAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 568, "pattern": "/" } ] }, { "agents": [ "LinerBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 571, "pattern": "/" } ] }, { "agents": [ "Poggio-Citations" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 574, "pattern": "/" } ] }, { "agents": [ "Devin" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 577, "pattern": "/" } ] }, { "agents": [ "bigsur.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 580, "pattern": "/" } ] }, { "agents": [ "ChatGPT Agent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 583, "pattern": "/" } ] }, { "agents": [ "GoogleAgent-Mariner" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 586, "pattern": "/" } ] }, { "agents": [ "AmazonBuyForMe" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 589, "pattern": "/" } ] }, { "agents": [ "Manus-User" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 592, "pattern": "/" } ] }, { "agents": [ "manus-user" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 595, "pattern": "/" } ] }, { "agents": [ "TwinAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 598, "pattern": "/" } ] }, { "agents": [ "AudigentAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 601, "pattern": "/" } ] }, { "agents": [ "AudigentBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 604, "pattern": "/" } ] }, { "agents": [ "JinaAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 607, "pattern": "/" } ] }, { "agents": [ "Jina-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 610, "pattern": "/" } ] }, { "agents": [ "JinaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 613, "pattern": "/" } ] }, { "agents": [ "Jina-Reader" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 616, "pattern": "/" } ] }, { "agents": [ "BrightData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 619, "pattern": "/" } ] }, { "agents": [ "Bright Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 622, "pattern": "/" } ] }, { "agents": [ "BrightBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 625, "pattern": "/" } ] }, { "agents": [ "SerpApi" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 628, "pattern": "/" } ] }, { "agents": [ "SerpApiBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 631, "pattern": "/" } ] }, { "agents": [ "Apify" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 634, "pattern": "/" } ] }, { "agents": [ "ApifyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 637, "pattern": "/" } ] }, { "agents": [ "Apify-HTTP-Client" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 640, "pattern": "/" } ] }, { "agents": [ "ScrapeGraphAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 643, "pattern": "/" } ] }, { "agents": [ "scrapegraph" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 646, "pattern": "/" } ] }, { "agents": [ "Scrapeless" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 649, "pattern": "/" } ] }, { "agents": [ "ScrapelessBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 652, "pattern": "/" } ] }, { "agents": [ "ZenRows" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 655, "pattern": "/" } ] }, { "agents": [ "ZenRowsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 658, "pattern": "/" } ] }, { "agents": [ "PromptCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 661, "pattern": "/" } ] }, { "agents": [ "PromptCloudBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 664, "pattern": "/" } ] }, { "agents": [ "Nimbleway" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 667, "pattern": "/" } ] }, { "agents": [ "NimbleCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 670, "pattern": "/" } ] }, { "agents": [ "NimblewayBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 673, "pattern": "/" } ] }, { "agents": [ "Oxylabs" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 676, "pattern": "/" } ] }, { "agents": [ "OxylabsBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 679, "pattern": "/" } ] }, { "agents": [ "Zyte" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 682, "pattern": "/" } ] }, { "agents": [ "ZyteBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 685, "pattern": "/" } ] }, { "agents": [ "Scrapinghub" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 688, "pattern": "/" } ] }, { "agents": [ "ScraperAPI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 691, "pattern": "/" } ] }, { "agents": [ "Scraper-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 694, "pattern": "/" } ] }, { "agents": [ "ScrapingBee" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 697, "pattern": "/" } ] }, { "agents": [ "ScrapingBeeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 700, "pattern": "/" } ] }, { "agents": [ "Scrapingdog" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 703, "pattern": "/" } ] }, { "agents": [ "ScrapingdogBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 706, "pattern": "/" } ] }, { "agents": [ "Grepsr" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 709, "pattern": "/" } ] }, { "agents": [ "GrepsrBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 712, "pattern": "/" } ] }, { "agents": [ "GrepsrCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 715, "pattern": "/" } ] }, { "agents": [ "SpiderBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 718, "pattern": "/" } ] }, { "agents": [ "spider-cloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 721, "pattern": "/" } ] }, { "agents": [ "SpiderCloud" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 724, "pattern": "/" } ] }, { "agents": [ "Hyperbrowser" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 727, "pattern": "/" } ] }, { "agents": [ "HyperbrowserBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 730, "pattern": "/" } ] }, { "agents": [ "HyperAgent" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 733, "pattern": "/" } ] }, { "agents": [ "Scrapestack" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 736, "pattern": "/" } ] }, { "agents": [ "ScrapestackBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 739, "pattern": "/" } ] }, { "agents": [ "WebScraping.AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 742, "pattern": "/" } ] }, { "agents": [ "WebScrapingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 745, "pattern": "/" } ] }, { "agents": [ "webscraping-ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 748, "pattern": "/" } ] }, { "agents": [ "ForageAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 751, "pattern": "/" } ] }, { "agents": [ "Forage-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 754, "pattern": "/" } ] }, { "agents": [ "ForageBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 757, "pattern": "/" } ] }, { "agents": [ "DumplingAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 760, "pattern": "/" } ] }, { "agents": [ "Dumpling-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 763, "pattern": "/" } ] }, { "agents": [ "DumplingBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 766, "pattern": "/" } ] }, { "agents": [ "Import.io" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 769, "pattern": "/" } ] }, { "agents": [ "ImportIO" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 772, "pattern": "/" } ] }, { "agents": [ "importio" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 775, "pattern": "/" } ] }, { "agents": [ "BrowseAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 778, "pattern": "/" } ] }, { "agents": [ "Browse-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 781, "pattern": "/" } ] }, { "agents": [ "BrowseAIBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 784, "pattern": "/" } ] }, { "agents": [ "Decodo" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 787, "pattern": "/" } ] }, { "agents": [ "DecodoBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 790, "pattern": "/" } ] }, { "agents": [ "Smartproxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 793, "pattern": "/" } ] }, { "agents": [ "Scrapfly" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 796, "pattern": "/" } ] }, { "agents": [ "ScrapflyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 799, "pattern": "/" } ] }, { "agents": [ "ScrapflyClient" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 802, "pattern": "/" } ] }, { "agents": [ "Octoparse" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 805, "pattern": "/" } ] }, { "agents": [ "OctoparseBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 808, "pattern": "/" } ] }, { "agents": [ "TrajectData" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 811, "pattern": "/" } ] }, { "agents": [ "Traject-Data" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 814, "pattern": "/" } ] }, { "agents": [ "SerpWow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 817, "pattern": "/" } ] }, { "agents": [ "ScaleSERP" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 820, "pattern": "/" } ] }, { "agents": [ "Rainforest-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 823, "pattern": "/" } ] }, { "agents": [ "BlueCart-API" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 826, "pattern": "/" } ] }, { "agents": [ "skrape.ai" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 829, "pattern": "/" } ] }, { "agents": [ "SkrapeAI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 832, "pattern": "/" } ] }, { "agents": [ "SkrapeBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 835, "pattern": "/" } ] }, { "agents": [ "Froxy" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 838, "pattern": "/" } ] }, { "agents": [ "FroxyBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 841, "pattern": "/" } ] }, { "agents": [ "MrScraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 844, "pattern": "/" } ] }, { "agents": [ "Mr-Scraper" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 847, "pattern": "/" } ] }, { "agents": [ "MrScraperBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 850, "pattern": "/" } ] }, { "agents": [ "CrawlNow" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 853, "pattern": "/" } ] }, { "agents": [ "CrawlNowBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 856, "pattern": "/" } ] }, { "agents": [ "Soax" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 859, "pattern": "/" } ] }, { "agents": [ "SoaxBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 862, "pattern": "/" } ] }, { "agents": [ "SOAX" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 865, "pattern": "/" } ] }, { "agents": [ "Kadoa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 868, "pattern": "/" } ] }, { "agents": [ "Kadoabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 871, "pattern": "/" } ] }, { "agents": [ "KadoaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 874, "pattern": "/" } ] }, { "agents": [ "ScrapeHero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 877, "pattern": "/" } ] }, { "agents": [ "Scrape-Hero" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 880, "pattern": "/" } ] }, { "agents": [ "ScrapeHeroBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 883, "pattern": "/" } ] }, { "agents": [ "Exa" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 886, "pattern": "/" } ] }, { "agents": [ "ExaBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 889, "pattern": "/" } ] }, { "agents": [ "Exa-AI" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 892, "pattern": "/" } ] }, { "agents": [ "ExaCrawler" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 895, "pattern": "/" } ] }, { "agents": [ "Exabot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 898, "pattern": "/" } ] }, { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 901, "pattern": "/errors" }, { "directive": "disallow", "line": 902, "pattern": "/interactive/" }, { "directive": "disallow", "line": 903, "pattern": "/userauth/" }, { "directive": "disallow", "line": 904, "pattern": "/ugc/" }, { "directive": "disallow", "line": 905, "pattern": "/feeds/" }, { "directive": "disallow", "line": 906, "pattern": "/services/" }, { "directive": "disallow", "line": 907, "pattern": "/facebook/" }, { "directive": "disallow", "line": 908, "pattern": "/version-info/" }, { "directive": "disallow", "line": 909, "pattern": "/longform/draft/" }, { "directive": "disallow", "line": 910, "pattern": "/story/draft/" }, { "directive": "disallow", "line": 911, "pattern": "/topic/*/smart/" }, { "directive": "disallow", "line": 912, "pattern": "/search" }, { "directive": "disallow", "line": 913, "pattern": "/module-showcase/" }, { "directive": "disallow", "line": 914, "pattern": "/newsletter/" }, { "directive": "disallow", "line": 915, "pattern": "/blended-newsletter/" }, { "directive": "disallow", "line": 916, "pattern": "/story/nletter/" }, { "directive": "disallow", "line": 917, "pattern": "/sports/services/photos/" }, { "directive": "disallow", "line": 918, "pattern": "/optimus" }, { "directive": "disallow", "line": 919, "pattern": "/ux-train" }, { "directive": "disallow", "line": 920, "pattern": "/story/advisory/" }, { "directive": "disallow", "line": 921, "pattern": "/.cam-tangent/" }, { "directive": "disallow", "line": 922, "pattern": "/pbd/" }, { "directive": "disallow", "line": 923, "pattern": "/gciaf/" }, { "directive": "disallow", "line": 924, "pattern": "/gcdn/gciaf/" }, { "directive": "disallow", "line": 925, "pattern": "/dcc/" }, { "directive": "disallow", "line": 926, "pattern": "/gcdn/dcc/" }, { "directive": "disallow", "line": 927, "pattern": "/dc/" }, { "directive": "disallow", "line": 928, "pattern": "/gcdn/dc/" }, { "directive": "disallow", "line": 929, "pattern": "/dcjs/" }, { "directive": "disallow", "line": 930, "pattern": "/gcdn/dcjs/" }, { "directive": "disallow", "line": 931, "pattern": "/content-queries/" }, { "directive": "disallow", "line": 932, "pattern": "/zxc/" }, { "directive": "disallow", "line": 933, "pattern": "/money/homefront/" }, { "directive": "disallow", "line": 934, "pattern": "/exp-cruise" }, { "directive": "disallow", "line": 935, "pattern": "/exp-las-vegas2" }, { "directive": "disallow", "line": 936, "pattern": "/exp-faw" }, { "directive": "disallow", "line": 937, "pattern": "/exp-caribbean" }, { "directive": "disallow", "line": 938, "pattern": "/exp-beach" }, { "directive": "disallow", "line": 939, "pattern": "/exp-cruise2" }, { "directive": "disallow", "line": 940, "pattern": "/yourtake" }, { "directive": "disallow", "line": 941, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-daniel-kelly-amfam/6661213/" }, { "directive": "disallow", "line": 942, "pattern": "/story/2014/03/20/ge-cfo-challenge-david-bartlett-amway/6653003/" }, { "directive": "disallow", "line": 943, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-art-mccarthy-neulion/6655521/" }, { "directive": "disallow", "line": 944, "pattern": "/story/sports/ncaab/2014/03/20/ge-cfo-challenge-david-gross-major-league-lacrosse/6646987/" }, { "directive": "disallow", "line": 945, "pattern": "/money/lookup/stocks/" }, { "directive": "disallow", "line": 946, "pattern": "/money/blueprint/archive/" }, { "directive": "disallow", "line": 947, "pattern": "/money/blueprint/transfer?pname=" } ] }, { "agents": [ "AmazonAdBot" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 950, "pattern": "/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://www.usatoday.com/news-sitemap.xml", "https://www.usatoday.com/web-sitemap-index.xml", "https://www.usatoday.com/video-sitemap-index.xml", "https://www.usatoday.com/money/blueprint/sitemap.xml", "https://www.usatoday.com/money/blueprint/sitemap-news.xml", "https://www.usatoday.com/online-betting/sitemap.xml", "https://www.usatoday.com/online-betting/news-sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Applebot-Extended" ], "rules": [ { "directive": "disallow", "line": 124, "pattern": "/" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "CCBot" ], "rules": [ { "directive": "disallow", "line": 61, "pattern": "/" } ], "state": "disallowed" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ChatGPT-User" ], "rules": [ { "directive": "disallow", "line": 34, "pattern": "/" }, { "directive": "allow", "line": 35, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 36, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 37, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 38, "pattern": "/shopping/" }, { "directive": "allow", "line": 39, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 40, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-SearchBot" ], "rules": [ { "directive": "disallow", "line": 148, "pattern": "/" } ], "state": "disallowed" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Claude-User" ], "rules": [ { "directive": "disallow", "line": 145, "pattern": "/" } ], "state": "disallowed" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "ClaudeBot" ], "rules": [ { "directive": "disallow", "line": 31, "pattern": "/" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Google-Extended" ], "rules": [ { "directive": "disallow", "line": 64, "pattern": "/" }, { "directive": "allow", "line": 65, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 66, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 67, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 68, "pattern": "/shopping/" }, { "directive": "allow", "line": 69, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 70, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "GPTBot" ], "rules": [ { "directive": "disallow", "line": 52, "pattern": "/" }, { "directive": "allow", "line": 53, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 54, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 55, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 56, "pattern": "/shopping/" }, { "directive": "allow", "line": 57, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 58, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "OAI-SearchBot" ], "rules": [ { "directive": "disallow", "line": 133, "pattern": "/" }, { "directive": "allow", "line": 134, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 135, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 136, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 137, "pattern": "/shopping/" }, { "directive": "allow", "line": 138, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 139, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "Perplexity-User" ], "rules": [ { "directive": "disallow", "line": 532, "pattern": "/" } ], "state": "disallowed" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "PerplexityBot" ], "rules": [ { "directive": "disallow", "line": 109, "pattern": "/" }, { "directive": "allow", "line": 110, "pattern": "/contributor-content/" }, { "directive": "allow", "line": 111, "pattern": "/story/special/contributor-content/" }, { "directive": "allow", "line": 112, "pattern": "/money/money-management/" }, { "directive": "allow", "line": 113, "pattern": "/shopping/" }, { "directive": "allow", "line": 114, "pattern": "/story/shopping/" }, { "directive": "allow", "line": 115, "pattern": "/story/sponsor-story/real-estate-ausa/2026/04/16/sell-your-home-for-top-dollar-by-beating-back-to-school-deadlines/89482853007/" } ], "state": "disallowed" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "2838e2f3-4189-44a4-b88f-7240cdee4174", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "983e2149315d65db69a5e0e064b3f9a367506b1a4c67bc9d469e03a7e71b5dda", "previousChainHash": "153694dc3ef498b3ba3e13c6864d5a8f1d0709aee61c743948e2562e573670cd" } ], "payPerCrawl": { "status": "unverified", "reason": "Authoritative discovery requires Cloudflare Web Bot Auth and the authenticated Discovery API." } }