{
  "summary": {
    "collectedAt": "2026-09-10T00:35:25.026Z",
    "method": "One GET of /robots.txt and one of /llms.txt per site, following redirects, 15s timeout. An agent counts as blocked only when a group naming that agent disallows / with no allow: / beside it.",
    "sites": 60,
    "robotsReachable": 57,
    "namingAtLeastOneAgent": 33,
    "blockingAtLeastOneAgent": 29,
    "llmsTxtPresent": 14,
    "perAgent": {
      "GPTBot": {
        "named": 24,
        "blocked": 17
      },
      "OAI-SearchBot": {
        "named": 13,
        "blocked": 9
      },
      "ChatGPT-User": {
        "named": 19,
        "blocked": 14
      },
      "ClaudeBot": {
        "named": 28,
        "blocked": 27
      },
      "Claude-Web": {
        "named": 19,
        "blocked": 18
      },
      "anthropic-ai": {
        "named": 23,
        "blocked": 21
      },
      "Google-Extended": {
        "named": 20,
        "blocked": 16
      },
      "CCBot": {
        "named": 27,
        "blocked": 25
      },
      "PerplexityBot": {
        "named": 24,
        "blocked": 22
      },
      "Perplexity-User": {
        "named": 14,
        "blocked": 14
      },
      "Bytespider": {
        "named": 28,
        "blocked": 27
      },
      "Applebot-Extended": {
        "named": 25,
        "blocked": 24
      },
      "meta-externalagent": {
        "named": 19,
        "blocked": 19
      },
      "Amazonbot": {
        "named": 21,
        "blocked": 17
      },
      "cohere-ai": {
        "named": 21,
        "blocked": 20
      }
    }
  },
  "rows": [
    {
      "host": "nytimes.com",
      "robotsStatus": 200,
      "robotsBytes": 8547,
      "agentsNamed": 15,
      "agentsBlocked": 14,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": false,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "washingtonpost.com",
      "robotsStatus": 200,
      "robotsBytes": 5804,
      "agentsNamed": 9,
      "agentsBlocked": 9,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 0
      }
    },
    {
      "host": "theguardian.com",
      "robotsStatus": 200,
      "robotsBytes": 2873,
      "agentsNamed": 8,
      "agentsBlocked": 8,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "bbc.co.uk",
      "robotsStatus": 200,
      "robotsBytes": 4907,
      "agentsNamed": 15,
      "agentsBlocked": 15,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "reuters.com",
      "robotsStatus": 200,
      "robotsBytes": 3908,
      "agentsNamed": 3,
      "agentsBlocked": 1,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": false,
        "ChatGPT-User": false,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": true,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 401
      }
    },
    {
      "host": "apnews.com",
      "robotsStatus": 200,
      "robotsBytes": 1293,
      "agentsNamed": 9,
      "agentsBlocked": 9,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": true,
        "meta-externalagent": null,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 403
      }
    },
    {
      "host": "bloomberg.com",
      "robotsStatus": 200,
      "robotsBytes": 7048,
      "agentsNamed": 14,
      "agentsBlocked": 14,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "wsj.com",
      "robotsStatus": 200,
      "robotsBytes": 4296,
      "agentsNamed": 3,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": false,
        "ChatGPT-User": false,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 403
      }
    },
    {
      "host": "ft.com",
      "robotsStatus": 200,
      "robotsBytes": 1556,
      "agentsNamed": 11,
      "agentsBlocked": 11,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": null,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 403
      }
    },
    {
      "host": "cnn.com",
      "robotsStatus": 200,
      "robotsBytes": 3456,
      "agentsNamed": 14,
      "agentsBlocked": 14,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": null,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "forbes.com",
      "robotsStatus": 200,
      "robotsBytes": 33649,
      "agentsNamed": 11,
      "agentsBlocked": 10,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": false,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "businessinsider.com",
      "robotsStatus": 200,
      "robotsBytes": 2231,
      "agentsNamed": 4,
      "agentsBlocked": 4,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "theverge.com",
      "robotsStatus": 200,
      "robotsBytes": 5650,
      "agentsNamed": 14,
      "agentsBlocked": 12,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": null,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": false,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "techcrunch.com",
      "robotsStatus": 200,
      "robotsBytes": 710,
      "agentsNamed": 10,
      "agentsBlocked": 10,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "arstechnica.com",
      "robotsStatus": 200,
      "robotsBytes": 1912,
      "agentsNamed": 12,
      "agentsBlocked": 12,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "wired.com",
      "robotsStatus": 200,
      "robotsBytes": 2224,
      "agentsNamed": 10,
      "agentsBlocked": 10,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "engadget.com",
      "robotsStatus": 200,
      "robotsBytes": 207,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 200
      }
    },
    {
      "host": "zdnet.com",
      "robotsStatus": 200,
      "robotsBytes": 5722,
      "agentsNamed": 14,
      "agentsBlocked": 14,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "cnet.com",
      "robotsStatus": 200,
      "robotsBytes": 13801,
      "agentsNamed": 14,
      "agentsBlocked": 14,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "theatlantic.com",
      "robotsStatus": 200,
      "robotsBytes": 3649,
      "agentsNamed": 15,
      "agentsBlocked": 12,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": false,
        "ChatGPT-User": false,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "vox.com",
      "robotsStatus": 200,
      "robotsBytes": 7548,
      "agentsNamed": 14,
      "agentsBlocked": 12,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": null,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": false,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "axios.com",
      "robotsStatus": 200,
      "robotsBytes": 645,
      "agentsNamed": 4,
      "agentsBlocked": 3,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": false,
        "CCBot": true,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": true,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "politico.com",
      "robotsStatus": 403,
      "robotsBytes": 0,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 403
      }
    },
    {
      "host": "npr.org",
      "robotsStatus": 0,
      "robotsBytes": 0,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 0
      }
    },
    {
      "host": "aljazeera.com",
      "robotsStatus": 200,
      "robotsBytes": 1834,
      "agentsNamed": 8,
      "agentsBlocked": 8,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "economist.com",
      "robotsStatus": 200,
      "robotsBytes": 2649,
      "agentsNamed": 12,
      "agentsBlocked": 12,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "newyorker.com",
      "robotsStatus": 200,
      "robotsBytes": 2188,
      "agentsNamed": 10,
      "agentsBlocked": 10,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "time.com",
      "robotsStatus": 200,
      "robotsBytes": 2326,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 716
      }
    },
    {
      "host": "nature.com",
      "robotsStatus": 200,
      "robotsBytes": 2721,
      "agentsNamed": 13,
      "agentsBlocked": 13,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": true,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "sciencedaily.com",
      "robotsStatus": 200,
      "robotsBytes": 87,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "searchengineland.com",
      "robotsStatus": 200,
      "robotsBytes": 170,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 1687620
      }
    },
    {
      "host": "searchenginejournal.com",
      "robotsStatus": 200,
      "robotsBytes": 2020,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 200
      }
    },
    {
      "host": "seroundtable.com",
      "robotsStatus": 200,
      "robotsBytes": 90,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "moz.com",
      "robotsStatus": 200,
      "robotsBytes": 1207,
      "agentsNamed": 1,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "ahrefs.com",
      "robotsStatus": 200,
      "robotsBytes": 908,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "semrush.com",
      "robotsStatus": 200,
      "robotsBytes": 3645,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 14948
      }
    },
    {
      "host": "backlinko.com",
      "robotsStatus": 200,
      "robotsBytes": 416,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "searchenginewatch.com",
      "robotsStatus": 200,
      "robotsBytes": 1836,
      "agentsNamed": 8,
      "agentsBlocked": 8,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 403
      }
    },
    {
      "host": "seopress.org",
      "robotsStatus": 200,
      "robotsBytes": 602,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 1319
      }
    },
    {
      "host": "yoast.com",
      "robotsStatus": 200,
      "robotsBytes": 927,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 7668
      }
    },
    {
      "host": "screamingfrog.co.uk",
      "robotsStatus": 200,
      "robotsBytes": 1883,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "sistrix.com",
      "robotsStatus": 200,
      "robotsBytes": 440,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "onely.com",
      "robotsStatus": 200,
      "robotsBytes": 9922,
      "agentsNamed": 7,
      "agentsBlocked": 6,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": false,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": true,
        "Google-Extended": null,
        "CCBot": true,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": true,
        "bytes": 1681
      }
    },
    {
      "host": "conductor.com",
      "robotsStatus": 200,
      "robotsBytes": 162,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 46992
      }
    },
    {
      "host": "brightedge.com",
      "robotsStatus": 200,
      "robotsBytes": 2512,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 1097
      }
    },
    {
      "host": "hubspot.com",
      "robotsStatus": 200,
      "robotsBytes": 12606,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 30688
      }
    },
    {
      "host": "shopify.com",
      "robotsStatus": 200,
      "robotsBytes": 62,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 12516
      }
    },
    {
      "host": "wordpress.org",
      "robotsStatus": 200,
      "robotsBytes": 976,
      "agentsNamed": 8,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": false,
        "Claude-Web": null,
        "anthropic-ai": false,
        "Google-Extended": false,
        "CCBot": false,
        "PerplexityBot": false,
        "Perplexity-User": null,
        "Bytespider": false,
        "Applebot-Extended": false,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 5763
      }
    },
    {
      "host": "cloudflare.com",
      "robotsStatus": 200,
      "robotsBytes": 1097,
      "agentsNamed": 8,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": false,
        "OAI-SearchBot": null,
        "ChatGPT-User": false,
        "ClaudeBot": null,
        "Claude-Web": false,
        "anthropic-ai": false,
        "Google-Extended": false,
        "CCBot": false,
        "PerplexityBot": false,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": false
      },
      "llmsTxt": {
        "present": true,
        "bytes": 17172
      }
    },
    {
      "host": "vercel.com",
      "robotsStatus": 200,
      "robotsBytes": 629,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 3797
      }
    },
    {
      "host": "stackoverflow.com",
      "robotsStatus": 418,
      "robotsBytes": 0,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "github.com",
      "robotsStatus": 200,
      "robotsBytes": 2274,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": true,
        "bytes": 28650
      }
    },
    {
      "host": "medium.com",
      "robotsStatus": 200,
      "robotsBytes": 884,
      "agentsNamed": 6,
      "agentsBlocked": 6,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": true,
        "Amazonbot": true,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 403
      }
    },
    {
      "host": "substack.com",
      "robotsStatus": 200,
      "robotsBytes": 585,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "quora.com",
      "robotsStatus": 200,
      "robotsBytes": 18792,
      "agentsNamed": 9,
      "agentsBlocked": 7,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": false,
        "CCBot": null,
        "PerplexityBot": true,
        "Perplexity-User": null,
        "Bytespider": true,
        "Applebot-Extended": true,
        "meta-externalagent": null,
        "Amazonbot": false,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "wikipedia.org",
      "robotsStatus": 200,
      "robotsBytes": 28027,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "reddit.com",
      "robotsStatus": 200,
      "robotsBytes": 538,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 200
      }
    },
    {
      "host": "linkedin.com",
      "robotsStatus": 200,
      "robotsBytes": 21308,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 200
      }
    },
    {
      "host": "amazon.com",
      "robotsStatus": 200,
      "robotsBytes": 7887,
      "agentsNamed": 12,
      "agentsBlocked": 12,
      "decisions": {
        "GPTBot": true,
        "OAI-SearchBot": true,
        "ChatGPT-User": true,
        "ClaudeBot": true,
        "Claude-Web": true,
        "anthropic-ai": null,
        "Google-Extended": true,
        "CCBot": true,
        "PerplexityBot": true,
        "Perplexity-User": true,
        "Bytespider": true,
        "Applebot-Extended": null,
        "meta-externalagent": true,
        "Amazonbot": null,
        "cohere-ai": true
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    },
    {
      "host": "etsy.com",
      "robotsStatus": 200,
      "robotsBytes": 52926,
      "agentsNamed": 0,
      "agentsBlocked": 0,
      "decisions": {
        "GPTBot": null,
        "OAI-SearchBot": null,
        "ChatGPT-User": null,
        "ClaudeBot": null,
        "Claude-Web": null,
        "anthropic-ai": null,
        "Google-Extended": null,
        "CCBot": null,
        "PerplexityBot": null,
        "Perplexity-User": null,
        "Bytespider": null,
        "Applebot-Extended": null,
        "meta-externalagent": null,
        "Amazonbot": null,
        "cohere-ai": null
      },
      "llmsTxt": {
        "present": false,
        "status": 404
      }
    }
  ]
}