{
  "version": "1.0.0",
  "generatedAt": "2026-08-13T02:20:03.237Z",
  "study": {
    "id": 1,
    "slug": "01-ai-crawlers",
    "state": "active_collection"
  },
  "window": {
    "start": "2026-07-15T21:20:49.000Z",
    "plannedEnd": "2026-08-14T21:20:49.000Z",
    "observedThrough": "2026-08-13T02:16:44.000Z",
    "elapsedDays": 28.21,
    "targetDays": 30,
    "progressPercent": 94
  },
  "totals": {
    "productionRequestsProcessed": 431815,
    "declaredCrawlerRequests": 116048,
    "verifiedRequests": 11502,
    "unverifiedRequests": 104546,
    "crawlerIdentities": 13
  },
  "crawlers": [
    {
      "crawler": "GPTBot",
      "operator": "OpenAI",
      "purpose": "model-development",
      "declaredRequests": 4884,
      "verifiedRequests": 4717,
      "unverifiedRequests": 167,
      "successfulResponses": 3992,
      "uniquePaths": 1641,
      "firstSeen": "2026-07-16T14:10:53.000Z",
      "lastSeen": "2026-08-13T00:00:30.000Z",
      "pageTypes": {
        "asset": 477,
        "other-page": 3469,
        "store": 126,
        "sitemap": 52,
        "home": 25,
        "journal-story-or-hub": 684,
        "services": 50,
        "vit-market": 1
      }
    },
    {
      "crawler": "Googlebot",
      "operator": "Google",
      "purpose": "traditional-search",
      "declaredRequests": 6625,
      "verifiedRequests": 2997,
      "unverifiedRequests": 3628,
      "successfulResponses": 5631,
      "uniquePaths": 1375,
      "firstSeen": "2026-07-15T21:25:57.000Z",
      "lastSeen": "2026-08-12T23:10:19.000Z",
      "pageTypes": {
        "robots": 728,
        "home": 54,
        "asset": 1499,
        "other-page": 2328,
        "journal-story-or-hub": 1787,
        "store": 145,
        "services": 43,
        "sitemap": 36,
        "vit-market": 5
      }
    },
    {
      "crawler": "OAI-SearchBot",
      "operator": "OpenAI",
      "purpose": "ai-search",
      "declaredRequests": 1893,
      "verifiedRequests": 1710,
      "unverifiedRequests": 183,
      "successfulResponses": 1693,
      "uniquePaths": 534,
      "firstSeen": "2026-07-15T22:46:27.000Z",
      "lastSeen": "2026-08-13T01:26:29.000Z",
      "pageTypes": {
        "robots": 135,
        "asset": 184,
        "other-page": 937,
        "store": 124,
        "journal-story-or-hub": 342,
        "services": 57,
        "home": 32,
        "sitemap": 17,
        "vit-market": 65
      }
    },
    {
      "crawler": "bingbot",
      "operator": "Microsoft",
      "purpose": "traditional-search",
      "declaredRequests": 1930,
      "verifiedRequests": 1699,
      "unverifiedRequests": 231,
      "successfulResponses": 1286,
      "uniquePaths": 941,
      "firstSeen": "2026-07-15T22:19:06.000Z",
      "lastSeen": "2026-08-13T01:42:25.000Z",
      "pageTypes": {
        "other-page": 956,
        "journal-story-or-hub": 513,
        "services": 8,
        "asset": 277,
        "store": 14,
        "robots": 58,
        "home": 18,
        "sitemap": 84,
        "vit-market": 2
      }
    },
    {
      "crawler": "ChatGPT-User",
      "operator": "OpenAI",
      "purpose": "user-directed",
      "declaredRequests": 947,
      "verifiedRequests": 379,
      "unverifiedRequests": 568,
      "successfulResponses": 653,
      "uniquePaths": 385,
      "firstSeen": "2026-07-15T23:48:07.000Z",
      "lastSeen": "2026-08-13T02:04:20.000Z",
      "pageTypes": {
        "sitemap": 8,
        "asset": 172,
        "other-page": 233,
        "home": 137,
        "services": 24,
        "journal-story-or-hub": 346,
        "store": 20,
        "robots": 1,
        "vit-market": 6
      }
    },
    {
      "crawler": "meta-externalagent",
      "operator": "Meta",
      "purpose": "model-development",
      "declaredRequests": 94084,
      "verifiedRequests": 0,
      "unverifiedRequests": 94084,
      "successfulResponses": 70201,
      "uniquePaths": 4160,
      "firstSeen": "2026-07-15T22:15:42.000Z",
      "lastSeen": "2026-08-13T02:16:44.000Z",
      "pageTypes": {
        "journal-story-or-hub": 16762,
        "other-page": 67603,
        "asset": 2950,
        "sitemap": 1028,
        "services": 1325,
        "store": 2958,
        "home": 725,
        "vit-market": 726,
        "research": 7
      }
    },
    {
      "crawler": "Amazonbot",
      "operator": "Amazon",
      "purpose": "model-development",
      "declaredRequests": 2868,
      "verifiedRequests": 0,
      "unverifiedRequests": 2868,
      "successfulResponses": 1523,
      "uniquePaths": 1630,
      "firstSeen": "2026-07-16T12:57:46.000Z",
      "lastSeen": "2026-08-13T02:14:36.000Z",
      "pageTypes": {
        "journal-story-or-hub": 548,
        "other-page": 1698,
        "asset": 531,
        "home": 25,
        "store": 44,
        "sitemap": 1,
        "services": 10,
        "robots": 10,
        "vit-market": 1
      }
    },
    {
      "crawler": "ClaudeBot",
      "operator": "Anthropic",
      "purpose": "model-development",
      "declaredRequests": 2450,
      "verifiedRequests": 0,
      "unverifiedRequests": 2450,
      "successfulResponses": 1932,
      "uniquePaths": 1765,
      "firstSeen": "2026-07-16T14:10:50.000Z",
      "lastSeen": "2026-08-13T01:38:21.000Z",
      "pageTypes": {
        "asset": 248,
        "other-page": 1545,
        "robots": 62,
        "home": 3,
        "sitemap": 66,
        "journal-story-or-hub": 361,
        "services": 7,
        "store": 35,
        "vit-market": 122,
        "research": 1
      }
    },
    {
      "crawler": "PerplexityBot",
      "operator": "Perplexity",
      "purpose": "ai-search",
      "declaredRequests": 230,
      "verifiedRequests": 0,
      "unverifiedRequests": 230,
      "successfulResponses": 115,
      "uniquePaths": 193,
      "firstSeen": "2026-07-16T14:10:51.000Z",
      "lastSeen": "2026-08-12T03:36:03.000Z",
      "pageTypes": {
        "asset": 96,
        "other-page": 73,
        "store": 3,
        "journal-story-or-hub": 31,
        "services": 8,
        "home": 5,
        "robots": 13,
        "vit-market": 1
      }
    },
    {
      "crawler": "CCBot",
      "operator": "Common Crawl",
      "purpose": "model-development",
      "declaredRequests": 102,
      "verifiedRequests": 0,
      "unverifiedRequests": 102,
      "successfulResponses": 57,
      "uniquePaths": 95,
      "firstSeen": "2026-07-16T14:10:53.000Z",
      "lastSeen": "2026-08-12T03:36:03.000Z",
      "pageTypes": {
        "asset": 37,
        "other-page": 44,
        "store": 5,
        "journal-story-or-hub": 15,
        "vit-market": 1
      }
    },
    {
      "crawler": "Perplexity-User",
      "operator": "Perplexity",
      "purpose": "user-directed",
      "declaredRequests": 17,
      "verifiedRequests": 0,
      "unverifiedRequests": 17,
      "successfulResponses": 0,
      "uniquePaths": 17,
      "firstSeen": "2026-07-19T02:47:48.000Z",
      "lastSeen": "2026-08-02T22:30:50.000Z",
      "pageTypes": {
        "other-page": 4,
        "asset": 13
      }
    },
    {
      "crawler": "Claude-User",
      "operator": "Anthropic",
      "purpose": "user-directed",
      "declaredRequests": 11,
      "verifiedRequests": 0,
      "unverifiedRequests": 11,
      "successfulResponses": 2,
      "uniquePaths": 11,
      "firstSeen": "2026-07-19T02:47:48.000Z",
      "lastSeen": "2026-07-22T15:36:53.000Z",
      "pageTypes": {
        "other-page": 4,
        "asset": 5,
        "robots": 1,
        "journal-story-or-hub": 1
      }
    },
    {
      "crawler": "Claude-SearchBot",
      "operator": "Anthropic",
      "purpose": "ai-search",
      "declaredRequests": 7,
      "verifiedRequests": 0,
      "unverifiedRequests": 7,
      "successfulResponses": 0,
      "uniquePaths": 7,
      "firstSeen": "2026-07-19T02:47:49.000Z",
      "lastSeen": "2026-07-19T02:47:52.000Z",
      "pageTypes": {
        "asset": 6,
        "other-page": 1
      }
    }
  ],
  "verification": {
    "enabled": true,
    "rule": "OpenAI and Google use official IP-range files retrieved at analysis time; Bing uses forward-confirmed reverse DNS. Unsupported identities remain unverified.",
    "sources": [
      {
        "crawler": "OAI-SearchBot",
        "url": "https://openai.com/searchbot.json",
        "createdAt": "2026-01-02T11:00:00.000000",
        "sha256": "e757b3ed5b84ac71976c02f17d9a77385e7acaf5b50e17c7b60f7e5a9876f102"
      },
      {
        "crawler": "OAI-AdsBot",
        "url": "https://openai.com/adsbot.json",
        "createdAt": "2026-05-12T03:00:00.000000",
        "sha256": "460b800ffb026c5ec1b9540d285f2ade8e1c4a654e65dae315e2bae650b5d74d"
      },
      {
        "crawler": "GPTBot",
        "url": "https://openai.com/gptbot.json",
        "createdAt": "2025-10-30T11:00:00.000000",
        "sha256": "be778b84e1b5436be6ab9d6139ba2b3d6a98c1f581e038ec386e3a7d44cbfe50"
      },
      {
        "crawler": "ChatGPT-User",
        "url": "https://openai.com/chatgpt-user.json",
        "createdAt": "2026-08-13T00:03:57.212440",
        "sha256": "6b0560e0560104a518a062de57a633e0f5af1454105d0454bf9252395156e5b4"
      },
      {
        "crawler": "Googlebot",
        "url": "https://developers.google.com/static/search/apis/ipranges/googlebot.json",
        "createdAt": "2026-08-12T14:46:35.000000",
        "sha256": "73780c089975b907afd1040605e23e0c1874904e30c504c2cb363b1bdcb641f1"
      },
      {
        "crawler": "bingbot",
        "url": "https://blogs.bing.com/webmaster/August-2012/How-to-Verify-that-Bingbot-is-Bingbot",
        "method": "forward-confirmed reverse DNS"
      }
    ]
  },
  "privacy": "No source IP, query value, raw user agent, request ID or individual visitor record is included in this public aggregate.",
  "publicationGate": "Collection is still in progress. This is an interim observation, not a completed 30-day result."
}