{
  "entity": {
    "id": 10,
    "slug": "googlebot",
    "vendor": "Google",
    "name": "Googlebot",
    "kind": "crawler",
    "purpose": "search",
    "ua_token": "Googlebot",
    "ua_pattern": "Googlebot",
    "robots_token": "Googlebot",
    "ip_list_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
    "docs_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
    "first_documented": "2026-09-13",
    "first_seen_here": null,
    "last_seen_here": null,
    "status": "active",
    "notes": "Common crawler for Google Search, Discover, Images, Video and News.",
    "created_at": "2026-09-13T00:00:00Z",
    "updated_at": "2026-09-13T00:00:00Z"
  },
  "claims": [
    {
      "id": 69,
      "entity_id": 10,
      "slug": "googlebot/cites_sources",
      "field": "cites_sources",
      "value": "yes",
      "statement": "Google AI Overviews and AI Mode return AI generated responses with links to supporting websites.",
      "evidence_url": "https://developers.google.com/search/docs/appearance/ai-features",
      "evidence_quote": "\"you can spend more time discovering ... get a comprehensive AI-powered response with links to supporting websites\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2025-12-10"
    },
    {
      "id": 64,
      "entity_id": 10,
      "slug": "googlebot/honours_crawl_delay",
      "field": "honours_crawl_delay",
      "value": "no",
      "statement": "Google does not support the robots.txt Crawl-delay directive for Googlebot.",
      "evidence_url": "https://developers.google.com/search/blog/2019/07/a-note-on-unsupported-rules-in-robotstxt",
      "evidence_quote": "\"Why isn't a code handler for other rules like crawl-delay included in the code?\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "medium",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": null
    },
    {
      "id": 65,
      "entity_id": 10,
      "slug": "googlebot/publishes_ip_list",
      "field": "publishes_ip_list",
      "value": "yes",
      "statement": "Google publishes the IP address ranges used by Googlebot at https://developers.google.com/static/crawling/ipranges/common-crawlers.json.",
      "evidence_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "evidence_quote": "\"match the crawler's IP address to the lists of Google crawlers' and fetchers' IP ranges\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-09-11"
    },
    {
      "id": 67,
      "entity_id": 10,
      "slug": "googlebot/purpose_documented",
      "field": "purpose_documented",
      "value": "search",
      "statement": "Googlebot crawls pages for Google Search, Google Images, Google Video, Google News, and Discover.",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": "\"Crawling preferences for Googlebot affect Google Search (including Discover and all Google Search features), as well as other products such as Google Images, Google Video, Google News, and Discover.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 132,
      "entity_id": 10,
      "slug": "googlebot/reads_llms_txt",
      "field": "reads_llms_txt",
      "value": "no",
      "statement": "Google Search does not use llms.txt files.",
      "evidence_url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide",
      "evidence_quote": "\"LLMS.txt files and other \"special\" markup: You don't need to create new machine readable files, AI text files, markup, or Markdown to appear in Google Search (including its generative AI capabilities), as Google Search itself doesn't use them.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-14",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:32:29Z",
      "updated_at": "2026-09-14T00:32:29Z",
      "evidence_date": "2026-07-10"
    },
    {
      "id": 63,
      "entity_id": 10,
      "slug": "googlebot/respects_robots_txt",
      "field": "respects_robots_txt",
      "value": "yes",
      "statement": "Googlebot obeys robots.txt rules when crawling automatically.",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": "\"They always obey robots.txt rules when crawling automatically.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 68,
      "entity_id": 10,
      "slug": "googlebot/separate_search_and_training_tokens",
      "field": "separate_search_and_training_tokens",
      "value": "yes",
      "statement": "Google separates Search crawling under the Googlebot token from Gemini training under the Google-Extended token.",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": "\"Google-Extended is a standalone product token that web publishers can use to manage whether content Google crawls from their sites may be used for training future generations of Gemini models\" and \"Google-Extended does not impact a site's inclusion in Google Search\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 70,
      "entity_id": 10,
      "slug": "googlebot/user_agent_string_full",
      "field": "user_agent_string_full",
      "value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "statement": "Googlebot sends the user agent string Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html).",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "evidence_quote": null,
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-07-14"
    },
    {
      "id": 66,
      "entity_id": 10,
      "slug": "googlebot/verifiable_by_rdns",
      "field": "verifiable_by_rdns",
      "value": "yes",
      "statement": "Googlebot requests are verifiable by reverse DNS lookup resolving to googlebot.com, google.com, or googleusercontent.com.",
      "evidence_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verifying-googlebot",
      "evidence_quote": "\"Run a reverse DNS lookup on the accessing IP address from your logs, using the host command\" and \"Verify that the domain name is either googlebot.com, google.com, or googleusercontent.com.\"",
      "method": "vendor_doc",
      "verified_at": "2026-09-13",
      "confidence": "high",
      "status": "current",
      "supersedes_id": null,
      "created_at": "2026-09-14T00:15:09Z",
      "updated_at": "2026-09-14T00:15:09Z",
      "evidence_date": "2026-03-20"
    }
  ],
  "observed": {
    "requests_30d": 0,
    "verified_share_30d": null,
    "daily_rows": []
  },
  "changes": [
    {
      "id": 132,
      "claim_id": 132,
      "entity_id": 10,
      "changed_at": "2026-09-14",
      "kind": "new",
      "old_value": null,
      "new_value": "no",
      "evidence_url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide",
      "note": "P0.3 googlebot llms.txt re-check batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 63,
      "claim_id": 63,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 64,
      "claim_id": 64,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "no",
      "evidence_url": "https://developers.google.com/search/blog/2019/07/a-note-on-unsupported-rules-in-robotstxt",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 65,
      "claim_id": 65,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 66,
      "claim_id": 66,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/crawling/docs/crawlers-fetchers/verifying-googlebot",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 67,
      "claim_id": 67,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "search",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 68,
      "claim_id": 68,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 69,
      "claim_id": 69,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "yes",
      "evidence_url": "https://developers.google.com/search/docs/appearance/ai-features",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    },
    {
      "id": 70,
      "claim_id": 70,
      "entity_id": 10,
      "changed_at": "2026-09-13",
      "kind": "new",
      "old_value": null,
      "new_value": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "evidence_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "P0.3 seed batch",
      "entity_slug": "googlebot",
      "entity_name": "Googlebot"
    }
  ],
  "notes": []
}