{
  "test": "quality-items",
  "title": "Quality of finished work: Item accuracy",
  "definition": "Share of required items that checked out among runs that handed in an answer, as reported in the published quality analysis.",
  "denominator": "Answered runs: 698 without USDC; 707 with USDC. These are run counts, not item denominators. Runs with no answer are excluded.",
  "snapshot": "2026-11-13 16:59 UTC",
  "scope": "Seven models, 21 tasks, five repeats per condition; 1,470 runs in total. Completion and cost give tasks and models equal weight.",
  "availability": "These are the published run aggregates, not individual run logs. The source snapshot does not include per-run records, prompts, outputs or traces. No individual run records have been reconstructed.",
  "units": "Scores are fractions from 0 to 1. Changes ending in _pp are percentage points; counts are eligible runs or claims as defined above.",
  "source": {
    "study": "/experiments/funded-agent-index",
    "methodology": "/experiments/funded-agent-index#spend-top"
  },
  "chartUrl": "/experiments/funded-agent-index?quality=items#quality",
  "downloads": {
    "json": "/experiments/funded-agent-index/data/quality-items.json",
    "csv": "/experiments/funded-agent-index/data/quality-items.csv"
  },
  "rows": [
    {
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_usdc": 0.5873192436040044,
      "with_usdc": 0.946524064171123,
      "change_pp": 35.920482056711855,
      "without_eligible_count": 103,
      "with_eligible_count": 105
    },
    {
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_usdc": 0.656353591160221,
      "with_usdc": 0.9240641711229947,
      "change_pp": 26.77105799627737,
      "without_eligible_count": 104,
      "with_eligible_count": 105
    },
    {
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_usdc": 0.41045606229143494,
      "with_usdc": 0.6844919786096256,
      "change_pp": 27.40359163181907,
      "without_eligible_count": 103,
      "with_eligible_count": 105
    },
    {
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_usdc": 0.7109905020352781,
      "with_usdc": 0.9392201834862385,
      "change_pp": 22.822968145096034,
      "without_eligible_count": 93,
      "with_eligible_count": 97
    },
    {
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_usdc": 0.5181518151815182,
      "with_usdc": 0.8144458281444583,
      "change_pp": 29.629401296294013,
      "without_eligible_count": 102,
      "with_eligible_count": 99
    },
    {
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_usdc": 0.45980707395498394,
      "with_usdc": 0.9561497326203209,
      "change_pp": 49.634265866533696,
      "without_eligible_count": 104,
      "with_eligible_count": 105
    },
    {
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_usdc": 0.6790830945558739,
      "with_usdc": 0.8975979772439949,
      "change_pp": 21.8514882688121,
      "without_eligible_count": 89,
      "with_eligible_count": 91
    },
    {
      "model": "All models (pooled)",
      "model_id": "all",
      "without_usdc": 0.5667224080267559,
      "with_usdc": 0.8807605543022881,
      "change_pp": 31.403814627553217,
      "without_eligible_count": 698,
      "with_eligible_count": 707
    }
  ],
  "taskRows": []
}