{
  "field": "honours_crawl_delay",
  "label": "honours crawl delay",
  "entities_documented": 15,
  "vendors_documented": 6,
  "claims": 15,
  "last_verified": "2026-09-14",
  "rows": [
    {
      "entity_slug": "applebot",
      "entity_name": "Applebot",
      "vendor": "Apple",
      "value": "no",
      "statement": "Applebot does not follow the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 10
    },
    {
      "entity_slug": "applebot-extended",
      "entity_name": "Applebot-Extended",
      "vendor": "Apple",
      "value": "n/a",
      "statement": "Apple does not publish a Crawl-delay behaviour for Applebot-Extended.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 2
    },
    {
      "entity_slug": "bingbot",
      "entity_name": "Bingbot",
      "vendor": "Microsoft",
      "value": "yes",
      "statement": "Microsoft states that a crawl-delay directive in robots.txt always takes precedence over Bing Webmaster Tools crawl control settings for Bingbot.",
      "method": "vendor_doc",
      "verified_at": "2026-09-14",
      "confidence": "high",
      "claim_id": 122
    },
    {
      "entity_slug": "chatgpt-user",
      "entity_name": "ChatGPT-User",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not publish whether ChatGPT-User honours the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 20
    },
    {
      "entity_slug": "claude-searchbot",
      "entity_name": "Claude-SearchBot",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Claude-SearchBot respects the Crawl-delay directive in robots.txt when crawling domains.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 29
    },
    {
      "entity_slug": "claude-user",
      "entity_name": "Claude-User",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Claude-User respects the Crawl-delay directive in robots.txt when fetching pages.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 39
    },
    {
      "entity_slug": "claudebot",
      "entity_name": "ClaudeBot",
      "vendor": "Anthropic",
      "value": "yes",
      "statement": "Anthropic supports the non-standard robots.txt Crawl-delay extension for ClaudeBot.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "claim_id": 48
    },
    {
      "entity_slug": "google-extended",
      "entity_name": "Google-Extended",
      "vendor": "Google",
      "value": "n/a",
      "statement": "Google does not publish any Crawl-delay behaviour for the Google-Extended token.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 56
    },
    {
      "entity_slug": "googlebot",
      "entity_name": "Googlebot",
      "vendor": "Google",
      "value": "no",
      "statement": "Google does not support the robots.txt Crawl-delay directive for Googlebot.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 64
    },
    {
      "entity_slug": "googleother",
      "entity_name": "GoogleOther",
      "vendor": "Google",
      "value": "no",
      "statement": "Google does not support the robots.txt Crawl-delay directive for GoogleOther.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "claim_id": 72
    },
    {
      "entity_slug": "gptbot",
      "entity_name": "GPTBot",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not publish whether GPTBot honours the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 79
    },
    {
      "entity_slug": "oai-adsbot",
      "entity_name": "OAI-AdsBot",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not publish whether OAI-AdsBot honours the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 87
    },
    {
      "entity_slug": "oai-searchbot",
      "entity_name": "OAI-SearchBot",
      "vendor": "OpenAI",
      "value": "not_documented",
      "statement": "OpenAI does not publish whether OAI-SearchBot honours the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 95
    },
    {
      "entity_slug": "perplexity-user",
      "entity_name": "Perplexity-User",
      "vendor": "Perplexity",
      "value": "not_documented",
      "statement": "Perplexity does not publish whether Perplexity-User honours the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 104
    },
    {
      "entity_slug": "perplexitybot",
      "entity_name": "PerplexityBot",
      "vendor": "Perplexity",
      "value": "not_documented",
      "statement": "Perplexity does not publish whether PerplexityBot honours the robots.txt Crawl-delay directive.",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": null,
      "claim_id": 113
    }
  ],
  "not_documented": [],
  "sources": [
    "https://support.apple.com/en-us/119829",
    "https://developers.openai.com/api/docs/bots",
    "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-content-from-the-web-and-how-can-site-owners-block-the-crawler",
    "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
    "https://developers.google.com/search/blog/2019/07/a-note-on-unsupported-rules-in-robotstxt",
    "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
    "https://www.bing.com/webmasters/help/crawl-control-55a30303"
  ]
}