{
  "field": "respects_robots_txt",
  "label": "respects robots txt",
  "entities_documented": 15,
  "vendors_documented": 6,
  "claims": 15,
  "last_verified": "2026-09-14",
  "rows": [
    {
      "entity_slug": "applebot",
      "entity_name": "Applebot",
      "vendor": "Apple",
      "value": "yes",
      "statement": "Applebot respects standard robots.txt directives targeted at Applebot in general search crawls.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 9
    },
    {
      "entity_slug": "applebot-extended",
      "entity_name": "Applebot-Extended",
      "vendor": "Apple",
      "value": "n/a",
      "statement": "Applebot-Extended is a robots.txt token that publishers disallow to opt out of generative model training.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 1
    },
    {
      "entity_slug": "bingbot",
      "entity_name": "Bingbot",
      "vendor": "Microsoft",
      "value": "yes",
      "statement": "Bingbot honors robots.txt directives and other Microsoft-supported control mechanisms that content owners set for crawling.",
      "method": "vendor_doc",
      "verified_at": "2026-09-14",
      "confidence": "high",
      "claim_id": 121
    },
    {
      "entity_slug": "chatgpt-user",
      "entity_name": "ChatGPT-User",
      "vendor": "OpenAI",
      "value": "partial",
      "statement": "ChatGPT-User fetches pages in response to user actions in ChatGPT, and OpenAI states that robots.txt rules may not apply to ChatGPT-User fetches.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 19
    },
    {
      "entity_slug": "claude-searchbot",
      "entity_name": "Claude-SearchBot",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Claude-SearchBot honours industry standard robots.txt directives that signal do not crawl.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 28
    },
    {
      "entity_slug": "claude-user",
      "entity_name": "Claude-User",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Claude-User honours industry standard robots.txt directives that signal do not crawl.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 37
    },
    {
      "entity_slug": "claudebot",
      "entity_name": "ClaudeBot",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "ClaudeBot honours industry standard robots.txt directives that signal do not crawl.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 47
    },
    {
      "entity_slug": "google-extended",
      "entity_name": "Google-Extended",
      "vendor": "Google",
      "value": "n/a",
      "statement": "Google-Extended is a robots.txt token that publishers set to control use of crawled content for Gemini training.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 55
    },
    {
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "vendor": "Google",
      "value": "yes",
      "statement": "Googlebot obeys robots.txt rules when crawling automatically.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 63
    },
    {
      "entity_slug": "googleother",
      "entity_name": "GoogleOther",
      "vendor": "Google",
      "value": "yes",
      "statement": "GoogleOther obeys robots.txt rules when crawling automatically.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 71
    },
    {
      "entity_slug": "gptbot",
      "entity_name": "GPTBot",
      "vendor": "OpenAI",
      "value": "yes",
      "statement": "GPTBot honours robots.txt rules that webmasters set for the GPTBot token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 78
    },
    {
      "entity_slug": "oai-adsbot",
      "entity_name": "OAI-AdsBot",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not publish whether OAI-AdsBot honours robots.txt disallow rules.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 86
    },
    {
      "entity_slug": "oai-searchbot",
      "entity_name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "value": "yes",
      "statement": "OAI-SearchBot honours robots.txt opt-outs set for the OAI-SearchBot token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 94
    },
    {
      "entity_slug": "perplexity-user",
      "entity_name": "Perplexity-User",
      "vendor": "Perplexity",
      "value": "no",
      "statement": "Perplexity-User ignores robots.txt rules because a user requests each Perplexity-User fetch.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 103
    },
    {
      "entity_slug": "perplexitybot",
      "entity_name": "PerplexityBot",
      "vendor": "Perplexity",
      "value": "yes",
      "statement": "PerplexityBot honours robots.txt rules that webmasters set for the PerplexityBot token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 112
    }
  ],
  "not_documented": [],
  "sources": [
    "https://support.apple.com/en-us/119829",
    "https://developers.openai.com/api/docs/bots",
    "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-content-from-the-web-and-how-can-site-owners-block-the-crawler",
    "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
    "https://www.bing.com/webmasters/help/ai-performance-9f8e7d6c"
  ]
}