{
  "capture_id": "2026-08-22T0815Z",
  "dataset_sha256": "6d6fe5b62eb7bd4c297fa6428653efa4b01d7abbf92792b7eee6a2cb813f4ca6",
  "manifest": {
    "schema_version": 1,
    "capture_id": "2026-08-22T0815Z",
    "capture_class": "pre-2026-09-15-baseline",
    "supersedes_metadata_of": "baseline-2026-08-22T0815Z-metadata-correction",
    "correction": "The first additive correction omitted content_type from its selected response-header value list. This final manifest exhaustively discloses the selected response-header metadata retained by the unchanged raw dataset: server, x_robots_tag, and content_type values; cf_ray presence; and allowlisted toll header names.",
    "observed_from": "2026-08-22T08:28:13.633Z",
    "observed_through": "2026-08-22T08:40:54.831Z",
    "row_run_value": "2026-08-22",
    "tag_target_commit": "f01808ee95843cb0ccc40989a942479a69466829",
    "tag_target_commit_time": "2026-08-22T08:43:09Z",
    "provenance_note": "The tag target records the timeout fix committed after the observations ended; the raw artifact hash and observed row timestamps are authoritative.",
    "input": {
      "name": "data/domains/tranco-74V8X-1000.csv",
      "tranco_list": "74V8X",
      "sha256": "f923fe95bc629b757b8599ad0910fec1f245d5b0355458772e1cffae1b7a1453"
    },
    "dataset": {
      "name": "2026-08-22T0815Z.jsonl",
      "sha256": "6d6fe5b62eb7bd4c297fa6428653efa4b01d7abbf92792b7eee6a2cb813f4ca6",
      "bytes": 3275141,
      "rows": 10000,
      "domains": 1000,
      "rows_per_domain": 10,
      "request_rows": 5000,
      "file_rows": 5000
    },
    "request_outcomes": {
      "reachable": 2384,
      "error": 1588,
      "challenged": 329,
      "denied_by_robots": 301,
      "blocked": 184,
      "client_error": 139,
      "other": 25,
      "rate_limited": 20,
      "toll": 15,
      "server_error": 14,
      "auth_required": 1
    },
    "privacy": {
      "response_bodies_stored": false,
      "cookies_stored": false,
      "presented_user_agent_strings_stored": false,
      "general_response_header_maps_stored": false,
      "selected_response_header_values_stored": [
        "server",
        "x_robots_tag",
        "content_type"
      ],
      "selected_header_presence_stored": [
        "cf_ray"
      ],
      "selected_header_names_stored": [
        "toll.headers allowlist"
      ]
    },
    "identity_note": "GPTBot and ClaudeBot rows are disclosed simulations, not authenticated vendor traffic; see METHODOLOGY.md at the tag target."
  },
  "aggregate": {
    "schema_versions": [
      1
    ],
    "rows": 10000,
    "domains": 1000,
    "vantages": [
      "unrecorded"
    ],
    "request_rows": 5000,
    "file_rows": 5000,
    "requests_sent": 4699,
    "reachable_rate_of_sent": 0.5073,
    "outcomes": {
      "denied_by_robots": 301,
      "error": 1588,
      "reachable": 2384,
      "client_error": 139,
      "blocked": 184,
      "other": 25,
      "challenged": 329,
      "server_error": 14,
      "rate_limited": 20,
      "auth_required": 1,
      "toll": 15
    },
    "by_dialect": {
      "browser": {
        "attempted": 1000,
        "sent": 969,
        "outcomes": {
          "denied_by_robots": 31,
          "error": 309,
          "reachable": 522,
          "client_error": 32,
          "blocked": 26,
          "other": 7,
          "challenged": 65,
          "auth_required": 1,
          "server_error": 3,
          "toll": 3,
          "rate_limited": 1
        }
      },
      "curl": {
        "attempted": 1000,
        "sent": 969,
        "outcomes": {
          "denied_by_robots": 31,
          "error": 314,
          "reachable": 500,
          "client_error": 29,
          "blocked": 42,
          "other": 5,
          "challenged": 69,
          "server_error": 4,
          "rate_limited": 3,
          "toll": 3
        }
      },
      "canireach": {
        "attempted": 1000,
        "sent": 969,
        "outcomes": {
          "denied_by_robots": 31,
          "error": 316,
          "reachable": 536,
          "client_error": 27,
          "blocked": 26,
          "other": 6,
          "challenged": 51,
          "rate_limited": 2,
          "server_error": 2,
          "toll": 3
        }
      },
      "gptbot": {
        "attempted": 1000,
        "sent": 897,
        "outcomes": {
          "denied_by_robots": 103,
          "error": 322,
          "reachable": 411,
          "client_error": 26,
          "blocked": 50,
          "other": 3,
          "challenged": 75,
          "rate_limited": 5,
          "server_error": 2,
          "toll": 3
        }
      },
      "claudebot": {
        "attempted": 1000,
        "sent": 895,
        "outcomes": {
          "denied_by_robots": 105,
          "error": 327,
          "reachable": 415,
          "client_error": 25,
          "blocked": 40,
          "other": 4,
          "challenged": 69,
          "rate_limited": 9,
          "server_error": 3,
          "toll": 3
        }
      }
    },
    "robots_policy": {
      "browser": {
        "allowed": 969,
        "denied": 31,
        "unknown": 0
      },
      "curl": {
        "allowed": 969,
        "denied": 31,
        "unknown": 0
      },
      "canireach": {
        "allowed": 969,
        "denied": 31,
        "unknown": 0
      },
      "gptbot": {
        "allowed": 897,
        "denied": 103,
        "unknown": 0
      },
      "claudebot": {
        "allowed": 895,
        "denied": 105,
        "unknown": 0
      }
    },
    "robots_txt": {
      "served_200": 575,
      "absent_or_error": 425
    },
    "redirects": {
      "observed": 0,
      "cross_origin": 0
    },
    "affordances": {
      "llms_txt": {
        "checked": 1000,
        "present": 102,
        "soft_404": 110,
        "not_checked": 39
      },
      "agents_md": {
        "checked": 1000,
        "present": 6,
        "soft_404": 119,
        "not_checked": 40
      },
      "wellknown_agents": {
        "checked": 1000,
        "present": 3,
        "soft_404": 101,
        "not_checked": 44
      },
      "web_bot_auth": {
        "checked": 1000,
        "present": 7,
        "soft_404": 102,
        "not_checked": 43
      }
    },
    "challenges": {
      "imperva-incapsula": 10,
      "cloudflare-challenge": 146,
      "akamai": 80,
      "datadome": 22,
      "recaptcha": 28,
      "aws-waf": 10,
      "hcaptcha": 18,
      "perimeterx": 15
    },
    "toll": {
      "status_402": 15,
      "crawler-price": 1
    },
    "top_servers": {
      "cloudflare": 737,
      "nginx": 443,
      "apache": 125,
      "amazons3": 113,
      "akamaighost": 109,
      "sffe": 70,
      "cloudfront": 64,
      "server": 57,
      "tengine": 53,
      "vercel": 43,
      "envoy": 42,
      "esf": 37
    }
  }
}