{
  "name": "ai-crawler-access",
  "title": "AI crawler access by domain",
  "description": "Which AI crawlers each domain disallows in robots.txt, measured directly. Domains whose robots.txt could not be established are included and flagged rather than dropped, so the denominator is the reader's to choose.",
  "homepage": "https://usestanding.com/research/ai-crawler-access",
  "licenses": [
    {
      "name": "CC-BY-4.0",
      "path": "https://creativecommons.org/licenses/by/4.0/"
    }
  ],
  "version": "2026-08-08",
  "population": {
    "domains_total": 54082,
    "robots_readable": 33670,
    "robots_unreadable": 20412,
    "blocking_at_least_one": 5497,
    "share_of_readable_blocking": 0.1633
  },
  "method": "One HTTPS request to /robots.txt per domain, parsed for Disallow rules applying to each named user-agent. No JavaScript, no login. A 404 counts as readable: there is no file, so nothing is disallowed there. A 403, a timeout or a TLS failure does not.",
  "agents": [
    {
      "user_agent": "GPTBot",
      "engine": "ChatGPT",
      "purpose": "trains and grounds OpenAI models"
    },
    {
      "user_agent": "OAI-SearchBot",
      "engine": "ChatGPT",
      "purpose": "builds ChatGPT's search index"
    },
    {
      "user_agent": "ChatGPT-User",
      "engine": "ChatGPT",
      "purpose": "fetches a page when a user's question needs it"
    },
    {
      "user_agent": "ClaudeBot",
      "engine": "Claude",
      "purpose": "trains and grounds Anthropic models"
    },
    {
      "user_agent": "Claude-User",
      "engine": "Claude",
      "purpose": "fetches a page during a Claude conversation"
    },
    {
      "user_agent": "Claude-SearchBot",
      "engine": "Claude",
      "purpose": "builds Claude's search index"
    },
    {
      "user_agent": "PerplexityBot",
      "engine": "Perplexity",
      "purpose": "builds Perplexity's index"
    },
    {
      "user_agent": "Perplexity-User",
      "engine": "Perplexity",
      "purpose": "fetches a page to answer a live question"
    },
    {
      "user_agent": "Google-Extended",
      "engine": "Gemini",
      "purpose": "controls Gemini grounding and training"
    },
    {
      "user_agent": "Googlebot",
      "engine": "Gemini / AI Overviews",
      "purpose": "feeds Google's index, including AI Overviews"
    }
  ],
  "resources": [
    {
      "name": "ai-crawler-access",
      "path": "ai-crawler-access.csv",
      "format": "csv"
    },
    {
      "name": "ai-crawler-access-jsonl",
      "path": "ai-crawler-access.jsonl",
      "format": "ndjson"
    }
  ]
}