{
  "field": "purpose_documented",
  "label": "purpose documented",
  "entities_documented": 15,
  "vendors_documented": 6,
  "claims": 15,
  "last_verified": "2026-09-14",
  "rows": [
    {
      "entity_slug": "applebot",
      "entity_name": "Applebot",
      "vendor": "Apple",
      "value": "mixed",
      "statement": "Applebot crawls content for Apple search results and for training Apple foundation models.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 13
    },
    {
      "entity_slug": "applebot-extended",
      "entity_name": "Applebot-Extended",
      "vendor": "Apple",
      "value": "policy_only",
      "statement": "Applebot-Extended controls whether website content is used to train Apple general purpose foundation models.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 5
    },
    {
      "entity_slug": "bingbot",
      "entity_name": "Bingbot",
      "vendor": "Microsoft",
      "value": "yes",
      "statement": "Bingbot serves as Microsoft's standard crawler and handles most of Bing's daily web crawling.",
      "method": "vendor_doc",
      "verified_at": "2026-09-14",
      "confidence": "high",
      "claim_id": 125
    },
    {
      "entity_slug": "chatgpt-user",
      "entity_name": "ChatGPT-User",
      "vendor": "OpenAI",
      "value": "user_fetch",
      "statement": "ChatGPT-User visits web pages for user actions in ChatGPT and in Custom GPTs.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 23
    },
    {
      "entity_slug": "claude-searchbot",
      "entity_name": "Claude-SearchBot",
      "vendor": "Anthropic",
      "value": "search",
      "statement": "Claude-SearchBot analyses online content to improve the relevance and accuracy of Claude search responses.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 32
    },
    {
      "entity_slug": "claude-user",
      "entity_name": "Claude-User",
      "vendor": "Anthropic",
      "value": "user_fetch",
      "statement": "Claude-User accesses websites in response to questions that individual users ask Claude.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 42
    },
    {
      "entity_slug": "claudebot",
      "entity_name": "ClaudeBot",
      "vendor": "Anthropic",
      "value": "training",
      "statement": "ClaudeBot collects web content for training Anthropic generative AI models.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 51
    },
    {
      "entity_slug": "google-extended",
      "entity_name": "Google-Extended",
      "vendor": "Google",
      "value": "policy_only",
      "statement": "Google-Extended is a standalone product token that controls whether Google uses crawled content to train Gemini models.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 59
    },
    {
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "vendor": "Google",
      "value": "search",
      "statement": "Googlebot crawls pages for Google Search, Google Images, Google Video, Google News, and Discover.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 67
    },
    {
      "entity_slug": "googleother",
      "entity_name": "GoogleOther",
      "vendor": "Google",
      "value": "mixed",
      "statement": "GoogleOther is a generic crawler that Google product teams use to fetch publicly accessible content from sites.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 75
    },
    {
      "entity_slug": "gptbot",
      "entity_name": "GPTBot",
      "vendor": "OpenAI",
      "value": "training",
      "statement": "GPTBot crawls content that may be used to train OpenAI generative AI foundation models.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 82
    },
    {
      "entity_slug": "oai-adsbot",
      "entity_name": "OAI-AdsBot",
      "vendor": "OpenAI",
      "value": "ads",
      "statement": "OAI-AdsBot visits landing pages submitted as ads on ChatGPT to check compliance with OpenAI policies.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 90
    },
    {
      "entity_slug": "oai-searchbot",
      "entity_name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "value": "search",
      "statement": "OAI-SearchBot surfaces websites in search results in ChatGPT search features.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 98
    },
    {
      "entity_slug": "perplexity-user",
      "entity_name": "Perplexity-User",
      "vendor": "Perplexity",
      "value": "user_fetch",
      "statement": "Perplexity-User visits web pages to answer questions that users ask Perplexity.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 107
    },
    {
      "entity_slug": "perplexitybot",
      "entity_name": "PerplexityBot",
      "vendor": "Perplexity",
      "value": "search",
      "statement": "PerplexityBot surfaces and links websites in Perplexity search results.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 116
    }
  ],
  "not_documented": [],
  "sources": [
    "https://support.apple.com/en-us/119829",
    "https://developers.openai.com/api/docs/bots",
    "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-content-from-the-web-and-how-can-site-owners-block-the-crawler",
    "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
    "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0"
  ]
}