{ "domain": "openai.com", "recordsNotConclusions": true, "events": [ { "id": "130f9a3e-897f-4b93-aa31-5d032f0b20aa", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-13T03:17:22.800Z", "httpStatus": 403, "responseHeaders": { "content-length": "9853", "content-type": "text/html; charset=UTF-8", "date": "Thu, 13 Aug 2026 03:17:22 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "984bbb506f763008a139ba5b0bda50bfbbc28d23f6609c1308567cfe9a668c84", "artifactBytes": 9853, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "130f9a3e-897f-4b93-aa31-5d032f0b20aa", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "5d2bb143ff282f6b3f4df7ee40336783bc6759cf325685b540d2fa870225de13", "previousChainHash": "4fbc67ad3d3eac1024a7c2c2790242dd08cf1f83d21d8921999564d6960b985b" }, { "id": "7b7521ed-68ec-4e86-9b99-974ebc1fb106", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-13T03:17:16.191Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Thu, 13 Aug 2026 03:17:16 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Thu, 16 Apr 2026 00:44:06 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6b0916897f05c0a58ee73ad8d6678c15ee71502b45634deff817b92848ec77dc", "previousChainHash": "bd4b26d4571eec4777a4c3a95a1dc35a52bd9aa0e74a9c15a71b65aadaabf466" }, { "id": "8206216a-5ab3-4483-bdb5-f4177068cf4c", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-12T03:17:16.907Z", "httpStatus": 403, "responseHeaders": { "content-length": "9831", "content-type": "text/html; charset=UTF-8", "date": "Wed, 12 Aug 2026 03:17:16 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "d4d008875cfcf20df5c4bf37aa3f4dc85255247712dff136a53f8e3404f5ceef", "artifactBytes": 9831, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "8206216a-5ab3-4483-bdb5-f4177068cf4c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "4fbc67ad3d3eac1024a7c2c2790242dd08cf1f83d21d8921999564d6960b985b", "previousChainHash": "efdf5381e73da9eed042d1693ce221c07462fb5e43d4af5e6b7c96e797f971d7" }, { "id": "3da19925-f23b-482f-8a75-4a6d5ea44eff", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-12T03:17:14.325Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Wed, 12 Aug 2026 03:17:14 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Thu, 16 Apr 2026 00:44:06 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "bd4b26d4571eec4777a4c3a95a1dc35a52bd9aa0e74a9c15a71b65aadaabf466", "previousChainHash": "22c4c0a79859bde3efd8871c99110b979ae0e7bd7863f8fb80eda4af8af1926c" }, { "id": "48a086b1-ec38-489d-b339-fe45b69fd645", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-11T03:17:16.368Z", "httpStatus": 403, "responseHeaders": { "content-length": "9853", "content-type": "text/html; charset=UTF-8", "date": "Tue, 11 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "5d45097fe39e213963a90ba13ac35e96e5aec99f1b573c6c194a6f0fec66b137", "artifactBytes": 9853, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "48a086b1-ec38-489d-b339-fe45b69fd645", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "efdf5381e73da9eed042d1693ce221c07462fb5e43d4af5e6b7c96e797f971d7", "previousChainHash": "63a2748add0022855dfe9419dbd9984a3a30a2697f2acf29a9c899b2633fc797" }, { "id": "9b8eae88-f8f3-41ce-8517-8b3a12961150", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-11T03:17:13.890Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Tue, 11 Aug 2026 03:17:13 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Wed, 22 Apr 2026 19:15:09 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "22c4c0a79859bde3efd8871c99110b979ae0e7bd7863f8fb80eda4af8af1926c", "previousChainHash": "6313c701d26aede8440503e666117981f5757f58f179526a6eba27e92a2e458b" }, { "id": "ee45483e-34af-43e3-9f6e-d6f99be6ccc3", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-10T03:17:16.661Z", "httpStatus": 403, "responseHeaders": { "content-length": "9831", "content-type": "text/html; charset=UTF-8", "date": "Mon, 10 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "0eb1914c1d7a7f67baf8472933959d6969f5d169aeb396e4e61595c05ab91f51", "artifactBytes": 9831, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "ee45483e-34af-43e3-9f6e-d6f99be6ccc3", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "63a2748add0022855dfe9419dbd9984a3a30a2697f2acf29a9c899b2633fc797", "previousChainHash": "c0630008bcdc083d5aedcaf3ca0a9b3119d7303d5a8cf8373db72a509aab8b4a" }, { "id": "5b4a59fd-5dcb-4a53-8951-46d6130397ea", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-10T03:17:14.184Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Mon, 10 Aug 2026 03:17:14 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Wed, 22 Apr 2026 19:15:09 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6313c701d26aede8440503e666117981f5757f58f179526a6eba27e92a2e458b", "previousChainHash": "a33c6344078e6f94b0767d0e8e3f4cf35212de616bbca7ebda2f9acd12fb86a8" }, { "id": "5eadc51b-ea71-4994-923b-ec3ff14db73b", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-09T03:17:16.193Z", "httpStatus": 403, "responseHeaders": { "content-length": "9853", "content-type": "text/html; charset=UTF-8", "date": "Sun, 09 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "a3b9c404057615ba36dd37bd318d5cf2638e2aa6c04616c29f0a19799fda8c00", "artifactBytes": 9853, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "5eadc51b-ea71-4994-923b-ec3ff14db73b", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c0630008bcdc083d5aedcaf3ca0a9b3119d7303d5a8cf8373db72a509aab8b4a", "previousChainHash": "6debef56769fa8bf6fe1757ccddf72b24c2734d58285fdedbf58bf889c914bac" }, { "id": "26aae0e4-0461-4b60-86b5-4493b9adff31", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-09T03:17:13.635Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Sun, 09 Aug 2026 03:17:13 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Thu, 16 Apr 2026 00:44:06 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "a33c6344078e6f94b0767d0e8e3f4cf35212de616bbca7ebda2f9acd12fb86a8", "previousChainHash": "3b55a3ca9f3f32b70b750da371efe04d652a01d13c5f22d29d98d46646d43c50" }, { "id": "af7364ab-e027-4ebb-b4a6-bc263fc4106c", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-08T03:17:16.711Z", "httpStatus": 403, "responseHeaders": { "content-length": "9853", "content-type": "text/html; charset=UTF-8", "date": "Sat, 08 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "68a03476ea3456b1dcc01be708617f6441e1ed043fe5b432eb8569b404254857", "artifactBytes": 9853, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "af7364ab-e027-4ebb-b4a6-bc263fc4106c", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "6debef56769fa8bf6fe1757ccddf72b24c2734d58285fdedbf58bf889c914bac", "previousChainHash": "d2b3faf20a53e9f22efcf43ff55898db785e79eaf652a7eee71d46d34e322de4" }, { "id": "6498f540-60dc-446f-b791-93a7cd8faaa9", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-08T03:17:14.148Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Sat, 08 Aug 2026 03:17:14 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Thu, 16 Apr 2026 00:44:06 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "3b55a3ca9f3f32b70b750da371efe04d652a01d13c5f22d29d98d46646d43c50", "previousChainHash": "7467cb221ceb42315a0d9c2f831fad7e52369398781be2ee98a70580c09026e9" }, { "id": "e43741ed-5fcd-412d-ab3d-d5c9e271434a", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-07T03:17:15.913Z", "httpStatus": 403, "responseHeaders": { "content-length": "9831", "content-type": "text/html; charset=UTF-8", "date": "Fri, 07 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "ed8ca3cb7e31dfb48f48c01cc39e9dbdf4d15c1d6aebbdc5fcbd0118942b7d05", "artifactBytes": 9831, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "e43741ed-5fcd-412d-ab3d-d5c9e271434a", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "d2b3faf20a53e9f22efcf43ff55898db785e79eaf652a7eee71d46d34e322de4", "previousChainHash": "598360fe655db501fc15ed18c45b1013c28bb55b2a2b0b4a66ee3a64098432f7" }, { "id": "199a4082-2a06-479c-9c51-aa7248f2000c", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-07T03:17:13.326Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Fri, 07 Aug 2026 03:17:13 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Wed, 22 Apr 2026 19:15:09 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "7467cb221ceb42315a0d9c2f831fad7e52369398781be2ee98a70580c09026e9", "previousChainHash": "632b2aa84d36591148e5e3c3649eed54d49d16119e0183093557dfd4e526303f" }, { "id": "4b206e33-d705-4e9a-9b29-37fcf2e7cb61", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-06T03:17:15.732Z", "httpStatus": 403, "responseHeaders": { "content-length": "9853", "content-type": "text/html; charset=UTF-8", "date": "Thu, 06 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "e404dbcbe42a8a2131f42bb8335d839bffa22f96607cd1f736e68dddd9f2a793", "artifactBytes": 9853, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "4b206e33-d705-4e9a-9b29-37fcf2e7cb61", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "598360fe655db501fc15ed18c45b1013c28bb55b2a2b0b4a66ee3a64098432f7", "previousChainHash": "4261f519e469a4a69bdce33566399f3c403dd1e059f6cc92e2ec7b8061109692" }, { "id": "b51d5089-10a1-46a0-ac87-3546ed44e80a", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-06T03:17:13.273Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Thu, 06 Aug 2026 03:17:13 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Wed, 22 Apr 2026 19:15:09 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "632b2aa84d36591148e5e3c3649eed54d49d16119e0183093557dfd4e526303f", "previousChainHash": "622cb37f2673547ef488bf418e42c3acc4c7eb8eb3b40c20c25529aba1eeadf7" }, { "id": "1d9e64cd-ceb6-40cf-bd7b-813e4cf50060", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-05T03:17:17.962Z", "httpStatus": 403, "responseHeaders": { "content-length": "9831", "content-type": "text/html; charset=UTF-8", "date": "Wed, 05 Aug 2026 03:17:17 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "ec5884566aa5f195609adce1ad888d413396cba0c01df10f0f3dcc07651c9c2d", "artifactBytes": 9831, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "1d9e64cd-ceb6-40cf-bd7b-813e4cf50060", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "4261f519e469a4a69bdce33566399f3c403dd1e059f6cc92e2ec7b8061109692", "previousChainHash": "9b3f8131e33a6d70d31c2432d88542d848089f8042e6882918c52fde5da09261" }, { "id": "1f8e19b4-5563-4629-a2d5-868f41d98c52", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-05T03:17:15.361Z", "httpStatus": 304, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "date": "Wed, 05 Aug 2026 03:17:15 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Thu, 16 Apr 2026 00:44:06 GMT", "server": "cloudflare" }, "parseStatus": "unchanged", "changed": false, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "622cb37f2673547ef488bf418e42c3acc4c7eb8eb3b40c20c25529aba1eeadf7", "previousChainHash": "c95f45f4102d7830296afd3399eaf1ac8d545ad519750fe4840d0c74e3dbe30a" }, { "id": "f031a142-7623-4600-8db3-fafb40b16406", "domain": "openai.com", "signal": "llms", "requestedUrl": "https://openai.com/llms.txt", "finalUrl": "https://openai.com/llms.txt", "fetchedAt": "2026-08-04T03:17:16.044Z", "httpStatus": 403, "responseHeaders": { "content-length": "9853", "content-type": "text/html; charset=UTF-8", "date": "Tue, 04 Aug 2026 03:17:15 GMT", "server": "cloudflare" }, "parseStatus": "http_error", "changed": true, "artifactSha256": "4d4f474f1c9fe419543eeb88c533f1ce5b0d6a785c23e4f8c002182d5233151c", "artifactBytes": 9853, "parserVersion": null, "parsed": null, "errorCode": "http_403", "canonicalEventId": "f031a142-7623-4600-8db3-fafb40b16406", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "9b3f8131e33a6d70d31c2432d88542d848089f8042e6882918c52fde5da09261", "previousChainHash": null }, { "id": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "domain": "openai.com", "signal": "robots", "requestedUrl": "https://openai.com/robots.txt", "finalUrl": "https://openai.com/robots.txt", "fetchedAt": "2026-08-04T03:17:13.059Z", "httpStatus": 200, "responseHeaders": { "cache-control": "public, max-age=0, must-revalidate", "content-length": "98", "content-type": "text/plain; charset=utf-8", "date": "Tue, 04 Aug 2026 03:17:13 GMT", "etag": "\"e32952d6049963aad2d1d7c44e1fa236\"", "last-modified": "Wed, 22 Apr 2026 19:15:09 GMT", "server": "cloudflare" }, "parseStatus": "parsed", "changed": true, "artifactSha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "artifactBytes": 98, "parserVersion": "robots-rfc9309-rsl-0.1.0", "parsed": { "globalLicenses": [], "groups": [ { "agents": [ "*" ], "crawlDelaySeconds": null, "licenses": [], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ] } ], "parserVersion": "robots-rfc9309-rsl-0.1.0", "registryVersion": "2026-08-03", "sitemaps": [ "https://openai.com/sitemap.xml" ], "trackedAgents": { "Applebot-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "CCBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ChatGPT-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Claude-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "ClaudeBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Google-Extended": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "GPTBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "OAI-SearchBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "Perplexity-User": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" }, "PerplexityBot": { "crawlDelaySeconds": null, "licenses": [], "matchedAgents": [ "*" ], "rules": [ { "directive": "allow", "line": 2, "pattern": "/" }, { "directive": "disallow", "line": 3, "pattern": "/microsoft-for-startups/" } ], "state": "partial" } }, "warnings": [] }, "errorCode": null, "canonicalEventId": "88b96639-ac4e-4f3d-bffb-cebf25c1c844", "ledgerFormatVersion": "crawlledger-event-0.1.0", "chainHash": "c95f45f4102d7830296afd3399eaf1ac8d545ad519750fe4840d0c74e3dbe30a", "previousChainHash": null } ] }