{ "domain": "medium.com", "recordsNotConclusions": true, "events": [ { "id": "380fd059-c731-4af5-8ef5-a5aaafb49dbf", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-13T11:17:16.071Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Thu, 13 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "22448ea987820ff9c0c609eae40cc47055a7551131ffb90d95398712ec7b7718", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "380fd059-c731-4af5-8ef5-a5aaafb49dbf", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d346aba4cd9ba79aa07b46b7e797d7b4d13cb091278aeef591ea4232a583a750", "previousChainHash": "b38562216887242aa2a080d194d135ef604008060430ce9543a7605fdd4b460b" }, { "id": "8bd21544-ce5c-417a-8ddb-3d85650e2ec8", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-13T11:17:13.383Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Thu, 13 Aug 2026 11:17:13 GMT", "last-modified": "Thu, 13 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "23bbd047db65019c9f331dea53e82f20478478ac01dabfba323334b2c85ecc79", "previousChainHash": "fb5c09601a745ca107e2e149dab24b7e5d69e0275bd613ade5d228fa369d5d5e" }, { "id": "4887b190-5bde-4ece-9926-a17f3e50d7cf", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-12T11:17:16.191Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Wed, 12 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "09959847bed4e6afff05cf153f537e91a466f762cb6a86a2dc93f6e0c76480f9", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "4887b190-5bde-4ece-9926-a17f3e50d7cf", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "b38562216887242aa2a080d194d135ef604008060430ce9543a7605fdd4b460b", "previousChainHash": "c1e67b71d549c6e7f5ba6f37ae200a09a1bdd141942d827234c99c7c1e0c927f" }, { "id": "71a24da8-c8e5-4479-bea1-5bb56cafc941", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-12T11:17:13.493Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Wed, 12 Aug 2026 11:17:13 GMT", "last-modified": "Wed, 12 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "fb5c09601a745ca107e2e149dab24b7e5d69e0275bd613ade5d228fa369d5d5e", "previousChainHash": "eb6ee8ef631b4c5db34276463bb10764be638f8b47a0b7b75f7ea46791aba308" }, { "id": "cd336292-d6ea-423e-a162-7c4b7fb5ddda", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-11T11:17:15.642Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Tue, 11 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "d21c201c795056ea02062c9ca12f579799681584d2a7f1856ea6e40e535b695c", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "cd336292-d6ea-423e-a162-7c4b7fb5ddda", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c1e67b71d549c6e7f5ba6f37ae200a09a1bdd141942d827234c99c7c1e0c927f", "previousChainHash": "7e4d6f0c8e4e35d79f05ab2c0c696e3836dee826bcf007165b88f522f35b024a" }, { "id": "c976b827-7e2a-477b-847c-32c2b5436d8c", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-11T11:17:13.033Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Tue, 11 Aug 2026 11:17:13 GMT", "last-modified": "Tue, 11 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "eb6ee8ef631b4c5db34276463bb10764be638f8b47a0b7b75f7ea46791aba308", "previousChainHash": "a6b1c3323837bb4692ede9cfa4d70d920837f3be6d73cc6ce97919021a2cc924" }, { "id": "40b6169c-6d31-47c5-88e7-70fcedb822b8", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-10T11:17:16.060Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Mon, 10 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "33632772fda8b6761cd9ad94fe24edfea68e5c89c2931e27a6aa27a94247bff0", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "40b6169c-6d31-47c5-88e7-70fcedb822b8", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7e4d6f0c8e4e35d79f05ab2c0c696e3836dee826bcf007165b88f522f35b024a", "previousChainHash": "f02fe399f993efad30316ae5731a396be70b684b12b7d74bc7b7bc75b88d1df9" }, { "id": "f573d1fa-aa89-4de2-b57a-05ddd81197e9", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-10T11:17:13.404Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Mon, 10 Aug 2026 11:17:13 GMT", "last-modified": "Mon, 10 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a6b1c3323837bb4692ede9cfa4d70d920837f3be6d73cc6ce97919021a2cc924", "previousChainHash": "39e0ca965a419b6f37b20ca4ac155bea1dc87cc70b03f4b635666cca83309c03" }, { "id": "d188338e-0e3b-48af-9211-0a40bcb579c2", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-09T11:17:16.046Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Sun, 09 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "0b461a42727fd438c645c87fa11c43534d44239847da4dfc5146411c855508ba", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "d188338e-0e3b-48af-9211-0a40bcb579c2", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "f02fe399f993efad30316ae5731a396be70b684b12b7d74bc7b7bc75b88d1df9", "previousChainHash": "6032b04f46c6f2b50c970521b7d0c9e842b18881be854af2f20de2f864c0724c" }, { "id": "8dcb2310-72ac-4795-878e-7d059c91703b", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-09T11:17:13.455Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Sun, 09 Aug 2026 11:17:13 GMT", "last-modified": "Sun, 09 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "39e0ca965a419b6f37b20ca4ac155bea1dc87cc70b03f4b635666cca83309c03", "previousChainHash": "e0b5f751fe49803c20a2e59cc7eb4bf3d3527c8faa9d4df02932bd3e5b41b62f" }, { "id": "0a346baa-3c38-4565-8a36-933875adcd53", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-08T11:17:16.179Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Sat, 08 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "4a63f5f2a129bfad49ff6c17d325592ffb19898214ba70ea25ddd6352dd6c82a", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "0a346baa-3c38-4565-8a36-933875adcd53", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6032b04f46c6f2b50c970521b7d0c9e842b18881be854af2f20de2f864c0724c", "previousChainHash": "57674534152156224ef32fed152a0eaee3f38299d833db3032a64925e067a83a" }, { "id": "d521c1ee-76e9-4907-9849-9899d2e32116", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-08T11:17:13.590Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Sat, 08 Aug 2026 11:17:13 GMT", "last-modified": "Sat, 08 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "e0b5f751fe49803c20a2e59cc7eb4bf3d3527c8faa9d4df02932bd3e5b41b62f", "previousChainHash": "58789224836102960065099f2b8301df42de3f9af7c58e5f616c0dfd70793cec" }, { "id": "8459bdd1-d1ad-4cba-bb8c-a51837e31863", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-07T11:17:16.029Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Fri, 07 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "fccc0cc6b5e4e9fc88e658f307843402ff0cb0783ea557a31ad4658d556dc4ab", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "8459bdd1-d1ad-4cba-bb8c-a51837e31863", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "57674534152156224ef32fed152a0eaee3f38299d833db3032a64925e067a83a", "previousChainHash": "0dd992e1f3586265cbfe518dbd59c0408f6880582638bbbc94c2803a0fe2153b" }, { "id": "403ebf2a-5566-4d1e-bf95-69d6c685ceb3", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-07T11:17:13.380Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Fri, 07 Aug 2026 11:17:13 GMT", "last-modified": "Fri, 07 Aug 2026 11:17:02 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "58789224836102960065099f2b8301df42de3f9af7c58e5f616c0dfd70793cec", "previousChainHash": "8bed7beca59f08e022cc154fbfa979e7885b23c4dbe80517e84247bc6cdb490e" }, { "id": "0f2c6c54-1da9-47bf-a0ff-0e0adbfb15d0", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-06T11:17:15.787Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Thu, 06 Aug 2026 11:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "1748343998ef9f61103e91b00b3e6c26278561e0f465dfd78a3846aee8c02311", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "0f2c6c54-1da9-47bf-a0ff-0e0adbfb15d0", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "0dd992e1f3586265cbfe518dbd59c0408f6880582638bbbc94c2803a0fe2153b", "previousChainHash": "15f4b92094b40e36df8dc43df95438ea5bd987a46914de90ffe9441822ae8f88" }, { "id": "87684454-0b15-4fa4-b4cf-1a82785300f9", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-06T11:17:13.057Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Thu, 06 Aug 2026 11:17:13 GMT", "last-modified": "Thu, 06 Aug 2026 11:17:13 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "8bed7beca59f08e022cc154fbfa979e7885b23c4dbe80517e84247bc6cdb490e", "previousChainHash": "343d3c55b5c31c8b01c83752cc2becb71b1fd51bc0ac3648857b1c66064fac93" }, { "id": "de80d0ea-6ce8-458d-adbd-39926934ff5d", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-05T11:17:15.863Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Wed, 05 Aug 2026 11:17:14 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "1cd276caf304fbffea6d26ca546927aaa0681dfebc66a8441669c553b86b28f5", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "de80d0ea-6ce8-458d-adbd-39926934ff5d", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "15f4b92094b40e36df8dc43df95438ea5bd987a46914de90ffe9441822ae8f88", "previousChainHash": "ee52b99534f971aaff9d3eda6e92cbcaa37828b1802c08e13d3eef45bea579ea" }, { "id": "670a689b-b6a9-459e-a942-4f44a962eb62", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-05T11:17:13.220Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Wed, 05 Aug 2026 11:17:13 GMT", "last-modified": "Wed, 05 Aug 2026 11:16:41 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": false, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "343d3c55b5c31c8b01c83752cc2becb71b1fd51bc0ac3648857b1c66064fac93", "previousChainHash": "ae346a8be53f283b793357a1f73b012fe3a351f1f2b763c9cae7e92084cbf74e" }, { "id": "cd8f9582-f33c-463c-9acf-560baed5e61a", "domain": "medium.com", "signal": "llms", "requestedUrl": "https://medium.com/llms.txt", "finalUrl": "https://medium.com/llms.txt", "fetchedAt": "2026-08-04T11:17:38.651Z", "httpStatus": 403, "responseHeaders": { "content-length": "5531", "content-type": "text/html; charset=UTF-8", "date": "Tue, 04 Aug 2026 11:17:37 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "a811f95845a6f6d3543111f7f035a37b4daef4b2583560b1f19cb46992a83eb1", "artifactBytes": 5531, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "cd8f9582-f33c-463c-9acf-560baed5e61a", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ee52b99534f971aaff9d3eda6e92cbcaa37828b1802c08e13d3eef45bea579ea", "previousChainHash": null }, { "id": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "domain": "medium.com", "signal": "robots", "requestedUrl": "https://medium.com/robots.txt", "finalUrl": "https://medium.com/robots.txt", "fetchedAt": "2026-08-04T11:17:28.318Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=7200", "content-length": "884", "content-type": "text/plain", "date": "Tue, 04 Aug 2026 11:17:28 GMT", "last-modified": "Tue, 04 Aug 2026 11:17:28 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "artifactBytes": 884, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ] }, { "agents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://medium.com/sitemap/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [ "https://medium.com/license.xml" ], "matchedAgents": [ "Amazonbot", "Applebot-Extended", "Bytespider", "ClaudeBot", "FacebookBot", "GoogleOther", "GPTBot", "meta-externalagent" ], "rules": [ { "directive": "disallow", "line": 30, "pattern": "/" }, { "directive": "allow", "line": 31, "pattern": "/about" }, { "directive": "allow", "line": 32, "pattern": "/business" }, { "directive": "allow", "line": 33, "pattern": "/earn" }, { "directive": "allow", "line": 34, "pattern": "/gift" }, { "directive": "allow", "line": 35, "pattern": "/membership" }, { "directive": "allow", "line": 36, "pattern": "/partner-program" }, { "directive": "allow", "line": 37, "pattern": "/verified-authors" } ], "state": "disallowed" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "disallow", "line": 2, "pattern": "/m/" }, { "directive": "disallow", "line": 3, "pattern": "/me/" }, { "directive": "disallow", "line": 4, "pattern": "/@me$" }, { "directive": "disallow", "line": 5, "pattern": "/@me/" }, { "directive": "disallow", "line": 6, "pattern": "/*/edit$" }, { "directive": "disallow", "line": 7, "pattern": "/*/*/edit$" }, { "directive": "disallow", "line": 8, "pattern": "/media/" }, { "directive": "disallow", "line": 9, "pattern": "/p/*/share" }, { "directive": "disallow", "line": 10, "pattern": "/r/" }, { "directive": "disallow", "line": 11, "pattern": "/trending" }, { "directive": "disallow", "line": 12, "pattern": "/search?q$" }, { "directive": "disallow", "line": 13, "pattern": "/search?q=" }, { "directive": "disallow", "line": 14, "pattern": "/*/search?q=" }, { "directive": "disallow", "line": 15, "pattern": "/*/search/*?q=" }, { "directive": "disallow", "line": 16, "pattern": "/*/*source=" }, { "directive": "allow", "line": 17, "pattern": "/_/api/users/*/meta" }, { "directive": "allow", "line": 18, "pattern": "/_/api/users/*/profile/stream" }, { "directive": "allow", "line": 19, "pattern": "/_/api/posts/*/responses" }, { "directive": "allow", "line": 20, "pattern": "/_/api/posts/*/responsesStream" }, { "directive": "allow", "line": 21, "pattern": "/_/api/posts/*/related" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "6c4c3fd3-218e-423e-933f-0f81b4f99b5f", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "ae346a8be53f283b793357a1f73b012fe3a351f1f2b763c9cae7e92084cbf74e", "previousChainHash": null } ] }