{
  "field": "separate_search_and_training_tokens",
  "label": "separate search and training tokens",
  "entities_documented": 15,
  "vendors_documented": 6,
  "claims": 15,
  "last_verified": "2026-09-14",
  "rows": [
    {
      "entity_slug": "applebot",
      "entity_name": "Applebot",
      "vendor": "Apple",
      "value": "partial",
      "statement": "Apple uses one crawler token, Applebot, for search crawling and for training crawling, and a second robots.txt token, Applebot-Extended, that opts content out of training use.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 14
    },
    {
      "entity_slug": "applebot-extended",
      "entity_name": "Applebot-Extended",
      "vendor": "Apple",
      "value": "yes",
      "statement": "Webpages that disallow Applebot-Extended remain eligible for inclusion in Apple search results.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 6
    },
    {
      "entity_slug": "bingbot",
      "entity_name": "Bingbot",
      "vendor": "Microsoft",
      "value": "no",
      "statement": "Microsoft controls Bingbot's use in AI training through meta tags rather than publishing a separate training-specific crawler token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-14",
      "confidence": "high",
      "claim_id": 126
    },
    {
      "entity_slug": "chatgpt-user",
      "entity_name": "ChatGPT-User",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not document ChatGPT-User as a search or training opt-out token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 24
    },
    {
      "entity_slug": "claude-searchbot",
      "entity_name": "Claude-SearchBot",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Anthropic separates search and training controls into two robots.txt tokens: Claude-SearchBot for search and ClaudeBot for training.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 33
    },
    {
      "entity_slug": "claude-user",
      "entity_name": "Claude-User",
      "vendor": "Anthropic",
      "value": "not_documented",
      "statement": "Anthropic does not document Claude-User as a search or training opt-out token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 43
    },
    {
      "entity_slug": "claudebot",
      "entity_name": "ClaudeBot",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Anthropic separates training and search controls into two robots.txt tokens: ClaudeBot for training and Claude-SearchBot for search.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 52
    },
    {
      "entity_slug": "google-extended",
      "entity_name": "Google-Extended",
      "vendor": "Google",
      "value": "yes",
      "statement": "Google-Extended controls Gemini training use only and does not affect inclusion in Google Search.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 60
    },
    {
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "vendor": "Google",
      "value": "yes",
      "statement": "Google separates Search crawling under the Googlebot token from Gemini training under the Google-Extended token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 68
    },
    {
      "entity_slug": "googleother",
      "entity_name": "GoogleOther",
      "vendor": "Google",
      "value": "not_documented",
      "statement": "Google does not document whether content fetched by GoogleOther is used for Gemini training.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 76
    },
    {
      "entity_slug": "gptbot",
      "entity_name": "GPTBot",
      "vendor": "OpenAI",
      "value": "yes",
      "statement": "OpenAI separates search and training controls into two robots.txt tokens: OAI-SearchBot for search and GPTBot for training.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 83
    },
    {
      "entity_slug": "oai-adsbot",
      "entity_name": "OAI-AdsBot",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not document OAI-AdsBot as part of a search and training token split.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 91
    },
    {
      "entity_slug": "oai-searchbot",
      "entity_name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "value": "yes",
      "statement": "OpenAI separates search and training controls into two robots.txt tokens: OAI-SearchBot for search and GPTBot for training.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 99
    },
    {
      "entity_slug": "perplexity-user",
      "entity_name": "Perplexity-User",
      "vendor": "Perplexity",
      "value": "not_documented",
      "statement": "Perplexity does not document Perplexity-User as a search or training opt-out token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 108
    },
    {
      "entity_slug": "perplexitybot",
      "entity_name": "PerplexityBot",
      "vendor": "Perplexity",
      "value": "not_documented",
      "statement": "Perplexity does not document a separate training crawler token alongside PerplexityBot.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 117
    }
  ],
  "not_documented": [],
  "sources": [
    "https://support.apple.com/en-us/119829",
    "https://developers.openai.com/api/docs/bots",
    "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-content-from-the-web-and-how-can-site-owners-block-the-crawler",
    "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
    "https://www.bing.com/webmasters/help/robots-meta-tags-and-attributes-that-bing-supports-5198d240"
  ]
}