{
  "test": "completion-cost",
  "title": "Completion vs. cost",
  "definition": "Mean cost includes model inference and services purchased by the agent, in USD per task. Checker purchases and web search are excluded. Completion is the share of predefined checks passed. Cost-change intervals are the published 95% ranges.",
  "denominator": "Seven models, 21 tasks, five repeats per condition; 1,470 runs in total. Completion and cost give tasks and models equal weight.",
  "snapshot": "2026-11-13 16:59 UTC",
  "scope": "Seven models, 21 tasks, five repeats per condition; 1,470 runs in total. Completion and cost give tasks and models equal weight.",
  "availability": "These are the published run aggregates, not individual run logs. The source snapshot does not include per-run records, prompts, outputs or traces. No individual run records have been reconstructed.",
  "units": "Costs are USD. Completion scores are fractions from 0 to 1; changes ending in _pp are percentage points.",
  "source": {
    "study": "/experiments/funded-agent-index",
    "methodology": "/experiments/funded-agent-index#spend-top"
  },
  "chartUrl": "/experiments/funded-agent-index#cost",
  "downloads": {
    "json": "/experiments/funded-agent-index/data/completion-cost.json",
    "csv": "/experiments/funded-agent-index/data/completion-cost.csv"
  },
  "taskPrivacyNote": "* some tasks have been slightly anonymized/modified to maintain privacy.",
  "rows": [
    {
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.6080352380952381,
      "with_completion": 0.9549971428571428,
      "completion_change_pp": 34.69619047619047,
      "without_cost_usd": 0.1673925619047619,
      "with_cost_usd": 0.22821788571428572,
      "extra_cost_usd": 0.060825323809523824,
      "extra_cost_low_usd": -0.015269852667732578,
      "extra_cost_high_usd": 0.1369205002867802,
      "with_agent_purchases_usd": 0.0652,
      "without_runs": 105,
      "with_runs": 105
    },
    {
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.5927657142857142,
      "with_completion": 0.9391742857142857,
      "completion_change_pp": 34.64085714285714,
      "without_cost_usd": 0.09006092380952382,
      "with_cost_usd": 0.1474636857142857,
      "extra_cost_usd": 0.05740276190476187,
      "extra_cost_low_usd": 0.00692135402909496,
      "extra_cost_high_usd": 0.10788416978042883,
      "with_agent_purchases_usd": 0.057280952380952384,
      "without_runs": 105,
      "with_runs": 105
    },
    {
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.4476409523809524,
      "with_completion": 0.67896,
      "completion_change_pp": 23.13190476190476,
      "without_cost_usd": 0.004790666666666667,
      "with_cost_usd": 0.03730325714285714,
      "extra_cost_usd": 0.03251259047619048,
      "extra_cost_low_usd": 0.007265187405587594,
      "extra_cost_high_usd": 0.05775999354679334,
      "with_agent_purchases_usd": 0.03228095238095238,
      "without_runs": 105,
      "with_runs": 105
    },
    {
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.531735238095238,
      "with_completion": 0.8692857142857142,
      "completion_change_pp": 33.755047619047616,
      "without_cost_usd": 0.24162973333333335,
      "with_cost_usd": 0.2215697142857143,
      "extra_cost_usd": -0.02006001904761906,
      "extra_cost_low_usd": -0.12175737583169674,
      "extra_cost_high_usd": 0.08163733773645862,
      "with_agent_purchases_usd": 0.06472095238095238,
      "without_runs": 105,
      "with_runs": 105
    },
    {
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.48378095238095237,
      "with_completion": 0.725112380952381,
      "completion_change_pp": 24.133142857142857,
      "without_cost_usd": 0.007380590476190476,
      "with_cost_usd": 0.06770930476190476,
      "extra_cost_usd": 0.060328714285714284,
      "extra_cost_low_usd": 0.01798316324135584,
      "extra_cost_high_usd": 0.1026742653300727,
      "with_agent_purchases_usd": 0.061605342857142864,
      "without_runs": 105,
      "with_runs": 105
    },
    {
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0.5600580952380952,
      "with_completion": 0.9475971428571428,
      "completion_change_pp": 38.753904761904764,
      "without_cost_usd": 0.05328951428571428,
      "with_cost_usd": 0.1452112380952381,
      "extra_cost_usd": 0.0919217238095238,
      "extra_cost_low_usd": 0.0381427164229752,
      "extra_cost_high_usd": 0.1457007311960724,
      "with_agent_purchases_usd": 0.0854705238095238,
      "without_runs": 105,
      "with_runs": 105
    },
    {
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.5366247619047618,
      "with_completion": 0.7984866666666667,
      "completion_change_pp": 26.186190476190475,
      "without_cost_usd": 0.14355755238095239,
      "with_cost_usd": 0.1834500857142857,
      "extra_cost_usd": 0.03989253333333331,
      "extra_cost_low_usd": -0.046251674078732985,
      "extra_cost_high_usd": 0.12603674074539964,
      "with_agent_purchases_usd": 0.07477192380952381,
      "without_runs": 105,
      "with_runs": 105
    }
  ],
  "taskRows": [
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0555556,
      "with_cost_usd": 0.13536359999999997,
      "extra_cost_usd": 0.07980799999999998,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.021287,
      "with_cost_usd": 0.056837000000000006,
      "extra_cost_usd": 0.035550000000000005,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 0.8,
      "completion_change_pp": 80,
      "without_cost_usd": 0.0010318,
      "with_cost_usd": 0.0224182,
      "extra_cost_usd": 0.0213864,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0,
      "with_completion": 0.8,
      "completion_change_pp": 80,
      "without_cost_usd": 0.0231726,
      "with_cost_usd": 0.1045798,
      "extra_cost_usd": 0.0814072,
      "with_agent_purchases_usd": 0.016,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 0.93334,
      "completion_change_pp": 93.33399999999999,
      "without_cost_usd": 0.0011802,
      "with_cost_usd": 0.02218,
      "extra_cost_usd": 0.0209998,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.012556600000000001,
      "with_cost_usd": 0.0484324,
      "extra_cost_usd": 0.0358758,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Email tomorrow's NYC forecast",
      "task_id": "cap-email-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0415504,
      "with_cost_usd": 0.052189,
      "extra_cost_usd": 0.010638599999999998,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.2,
      "with_completion": 1,
      "completion_change_pp": 80,
      "without_cost_usd": 0.021736600000000002,
      "with_cost_usd": 0.1043816,
      "extra_cost_usd": 0.082645,
      "with_agent_purchases_usd": 0.0615,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.2,
      "with_completion": 0.8,
      "completion_change_pp": 60.00000000000001,
      "without_cost_usd": 0.0101292,
      "with_cost_usd": 0.1575878,
      "extra_cost_usd": 0.1474586,
      "with_agent_purchases_usd": 0.1192,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.2,
      "with_completion": 0.4,
      "completion_change_pp": 20,
      "without_cost_usd": 0.0005704,
      "with_cost_usd": 0.1619292,
      "extra_cost_usd": 0.1613588,
      "with_agent_purchases_usd": 0.15940000000000001,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0034298,
      "with_cost_usd": 0.14383980000000002,
      "extra_cost_usd": 0.14041,
      "with_agent_purchases_usd": 0.0484,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.000129,
      "with_cost_usd": 0.0978684,
      "extra_cost_usd": 0.09773939999999999,
      "with_agent_purchases_usd": 0.09506060000000001,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0028628,
      "with_cost_usd": 0.1581862,
      "extra_cost_usd": 0.1553234,
      "with_agent_purchases_usd": 0.112941,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Photo of a forest sign",
      "task_id": "cap-image-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0057112,
      "with_cost_usd": 0.16520859999999998,
      "extra_cost_usd": 0.15949739999999998,
      "with_agent_purchases_usd": 0.12789040000000002,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0227632,
      "with_cost_usd": 0.1988884,
      "extra_cost_usd": 0.17612519999999998,
      "with_agent_purchases_usd": 0.1595,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.007879,
      "with_cost_usd": 0.1789988,
      "extra_cost_usd": 0.17111980000000002,
      "with_agent_purchases_usd": 0.1595,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0003882,
      "with_cost_usd": 0.1608004,
      "extra_cost_usd": 0.1604122,
      "with_agent_purchases_usd": 0.1595,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.0056304,
      "with_cost_usd": 0.2926738,
      "extra_cost_usd": 0.2870434,
      "with_agent_purchases_usd": 0.1835,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 0.8,
      "completion_change_pp": 80,
      "without_cost_usd": 0.00023280000000000002,
      "with_cost_usd": 0.1795378,
      "extra_cost_usd": 0.179305,
      "with_agent_purchases_usd": 0.1784516,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.005076,
      "with_cost_usd": 0.1746452,
      "extra_cost_usd": 0.1695692,
      "with_agent_purchases_usd": 0.1595,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Hat-trick song for the drive home",
      "task_id": "cap-song-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.008501,
      "with_cost_usd": 0.168692,
      "extra_cost_usd": 0.160191,
      "with_agent_purchases_usd": 0.1595,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0,
      "with_completion": 0.9666600000000001,
      "completion_change_pp": 96.666,
      "without_cost_usd": 0.079191,
      "with_cost_usd": 0.4303928,
      "extra_cost_usd": 0.3512018,
      "with_agent_purchases_usd": 0.064,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0,
      "with_completion": 0.4,
      "completion_change_pp": 40,
      "without_cost_usd": 0.0341946,
      "with_cost_usd": 0.1764698,
      "extra_cost_usd": 0.14227520000000002,
      "with_agent_purchases_usd": 0.04,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 0.2,
      "completion_change_pp": 20,
      "without_cost_usd": 0.001279,
      "with_cost_usd": 0.0082884,
      "extra_cost_usd": 0.0070094,
      "with_agent_purchases_usd": 0.004,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0,
      "with_completion": 0.8,
      "completion_change_pp": 80,
      "without_cost_usd": 0.30077980000000004,
      "with_cost_usd": 0.5045204,
      "extra_cost_usd": 0.20374059999999994,
      "with_agent_purchases_usd": 0.05600000000000001,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 0.4,
      "completion_change_pp": 40,
      "without_cost_usd": 0.0018004,
      "with_cost_usd": 0.0850098,
      "extra_cost_usd": 0.0832094,
      "with_agent_purchases_usd": 0.08,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.009135,
      "with_cost_usd": 0.24907680000000001,
      "extra_cost_usd": 0.2399418,
      "with_agent_purchases_usd": 0.13599999999999998,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest TikTok videos and their views",
      "task_id": "cap-tiktok-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.1,
      "with_completion": 0.4,
      "completion_change_pp": 30.000000000000004,
      "without_cost_usd": 0.1549158,
      "with_cost_usd": 0.2402756,
      "extra_cost_usd": 0.08535980000000001,
      "with_agent_purchases_usd": 0.096,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.4,
      "with_completion": 1,
      "completion_change_pp": 60,
      "without_cost_usd": 0.1608868,
      "with_cost_usd": 0.3664774,
      "extra_cost_usd": 0.2055906,
      "with_agent_purchases_usd": 0.15,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.96,
      "with_completion": 0.9199999999999999,
      "completion_change_pp": -3.9999999999999925,
      "without_cost_usd": 0.24187340000000002,
      "with_cost_usd": 0.3183396,
      "extra_cost_usd": 0.07646619999999998,
      "with_agent_purchases_usd": 0.091,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.08666,
      "with_completion": 0.8,
      "completion_change_pp": 71.334,
      "without_cost_usd": 0.005984,
      "with_cost_usd": 0.12859199999999998,
      "extra_cost_usd": 0.12260799999999998,
      "with_agent_purchases_usd": 0.121,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.6,
      "with_completion": 1,
      "completion_change_pp": 40,
      "without_cost_usd": 0.492552,
      "with_cost_usd": 0.2797018,
      "extra_cost_usd": -0.2128502,
      "with_agent_purchases_usd": 0.15,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.30666000000000004,
      "with_completion": 1,
      "completion_change_pp": 69.334,
      "without_cost_usd": 0.0070878,
      "with_cost_usd": 0.1556078,
      "extra_cost_usd": 0.14851999999999999,
      "with_agent_purchases_usd": 0.151,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0.13332000000000002,
      "with_completion": 1,
      "completion_change_pp": 86.668,
      "without_cost_usd": 0.058407799999999996,
      "with_cost_usd": 0.2244586,
      "extra_cost_usd": 0.1660508,
      "with_agent_purchases_usd": 0.15,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 space accounts on X",
      "task_id": "cap-followers-v2",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.38,
      "with_completion": 0.6,
      "completion_change_pp": 21.999999999999996,
      "without_cost_usd": 0.3501058,
      "with_cost_usd": 0.1584362,
      "extra_cost_usd": -0.19166960000000002,
      "with_agent_purchases_usd": 0.05700000000000001,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.2,
      "with_completion": 1,
      "completion_change_pp": 80,
      "without_cost_usd": 0.0900984,
      "with_cost_usd": 0.24463,
      "extra_cost_usd": 0.1545316,
      "with_agent_purchases_usd": 0.0184,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0,
      "with_completion": 0.93334,
      "completion_change_pp": 93.33399999999999,
      "without_cost_usd": 0.014916200000000001,
      "with_cost_usd": 0.1778868,
      "extra_cost_usd": 0.16297060000000002,
      "with_agent_purchases_usd": 0.024,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 0,
      "completion_change_pp": 0,
      "without_cost_usd": 0.002382,
      "with_cost_usd": 0.082664,
      "extra_cost_usd": 0.080282,
      "with_agent_purchases_usd": 0.08,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.2,
      "with_completion": 0,
      "completion_change_pp": -20,
      "without_cost_usd": 0.1214072,
      "with_cost_usd": 0.10054199999999999,
      "extra_cost_usd": -0.020865200000000014,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 0.93334,
      "completion_change_pp": 93.33399999999999,
      "without_cost_usd": 0.005495999999999999,
      "with_cost_usd": 0.012643999999999999,
      "extra_cost_usd": 0.007148,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.005057000000000001,
      "with_cost_usd": 0.0934808,
      "extra_cost_usd": 0.0884238,
      "with_agent_purchases_usd": 0.0268,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "NASA's latest posts on X, with live counts",
      "task_id": "cap-twitter-v2",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.2,
      "with_completion": 0.46668000000000004,
      "completion_change_pp": 26.667999999999996,
      "without_cost_usd": 0.2062432,
      "with_cost_usd": 0.53248,
      "extra_cost_usd": 0.3262368,
      "with_agent_purchases_usd": 0.1847,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 0.8,
      "completion_change_pp": -19.999999999999996,
      "without_cost_usd": 0.1059634,
      "with_cost_usd": 0.067939,
      "extra_cost_usd": -0.0380244,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.72,
      "with_completion": 1,
      "completion_change_pp": 28.000000000000004,
      "without_cost_usd": 0.0596062,
      "with_cost_usd": 0.0443602,
      "extra_cost_usd": -0.015245999999999996,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.52,
      "with_completion": 0.6,
      "completion_change_pp": 7.9999999999999964,
      "without_cost_usd": 0.0043184,
      "with_cost_usd": 0.0118436,
      "extra_cost_usd": 0.007525199999999999,
      "with_agent_purchases_usd": 0.008,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.4,
      "with_completion": 0.8,
      "completion_change_pp": 40,
      "without_cost_usd": 0.1138536,
      "with_cost_usd": 0.0997972,
      "extra_cost_usd": -0.014056399999999997,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.72,
      "with_completion": 0.8,
      "completion_change_pp": 8.000000000000007,
      "without_cost_usd": 0.0028524,
      "with_cost_usd": 0.0177598,
      "extra_cost_usd": 0.0149074,
      "with_agent_purchases_usd": 0.014000000000000002,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 0.2,
      "completion_change_pp": -80,
      "without_cost_usd": 0.016256399999999997,
      "with_cost_usd": 0.04245,
      "extra_cost_usd": 0.026193600000000004,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "AA 100 flight status",
      "task_id": "cap-flight-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 1,
      "with_completion": 0.8,
      "completion_change_pp": -19.999999999999996,
      "without_cost_usd": 0.1581356,
      "with_cost_usd": 0.06103879999999999,
      "extra_cost_usd": -0.0970968,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 0.9428599999999999,
      "completion_change_pp": -5.713999999999997,
      "without_cost_usd": 0.22593960000000002,
      "with_cost_usd": 0.2718856,
      "extra_cost_usd": 0.04594599999999999,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.97142,
      "with_completion": 0.97142,
      "completion_change_pp": 0,
      "without_cost_usd": 0.11266659999999999,
      "with_cost_usd": 0.10544139999999999,
      "extra_cost_usd": -0.007225200000000001,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.6571400000000001,
      "with_completion": 0.62856,
      "completion_change_pp": -2.858000000000005,
      "without_cost_usd": 0.0056108,
      "with_cost_usd": 0.006668200000000001,
      "extra_cost_usd": 0.0010574000000000009,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.97142,
      "with_completion": 1,
      "completion_change_pp": 2.858000000000005,
      "without_cost_usd": 0.2540262,
      "with_cost_usd": 0.2694974,
      "extra_cost_usd": 0.015471200000000018,
      "with_agent_purchases_usd": 0.00504,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.8285600000000001,
      "with_completion": 0.8856999999999999,
      "completion_change_pp": 5.7140000000000075,
      "without_cost_usd": 0.004063,
      "with_cost_usd": 0.0041256,
      "extra_cost_usd": 0.00006260000000000033,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0617946,
      "with_cost_usd": 0.12185939999999999,
      "extra_cost_usd": 0.060064799999999995,
      "with_agent_purchases_usd": 0.046040000000000005,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Who [company name redacted] is hiring right now*",
      "task_id": "cap-hiring-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.9428599999999999,
      "with_completion": 0.6857,
      "completion_change_pp": -25.716000000000005,
      "without_cost_usd": 0.093701,
      "with_cost_usd": 0.0809258,
      "extra_cost_usd": -0.0127752,
      "with_agent_purchases_usd": 0.00252,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.8,
      "with_completion": 1,
      "completion_change_pp": 19.999999999999996,
      "without_cost_usd": 0.28714080000000003,
      "with_cost_usd": 0.0778218,
      "extra_cost_usd": -0.20931900000000003,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.8,
      "with_completion": 1,
      "completion_change_pp": 19.999999999999996,
      "without_cost_usd": 0.15351520000000002,
      "with_cost_usd": 0.0617622,
      "extra_cost_usd": -0.09175300000000002,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.4,
      "with_completion": 1,
      "completion_change_pp": 60,
      "without_cost_usd": 0.0080012,
      "with_cost_usd": 0.015619,
      "extra_cost_usd": 0.007617799999999999,
      "with_agent_purchases_usd": 0.012,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.8,
      "with_completion": 1,
      "completion_change_pp": 19.999999999999996,
      "without_cost_usd": 0.2245784,
      "with_cost_usd": 0.09206980000000001,
      "extra_cost_usd": -0.1325086,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.9199999999999999,
      "with_completion": 1,
      "completion_change_pp": 7.9999999999999964,
      "without_cost_usd": 0.009110400000000001,
      "with_cost_usd": 0.014285,
      "extra_cost_usd": 0.0051746,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0.8800000000000001,
      "with_completion": 0.96,
      "completion_change_pp": 7.9999999999999964,
      "without_cost_usd": 0.15179700000000002,
      "with_cost_usd": 0.11792240000000001,
      "extra_cost_usd": -0.033874600000000005,
      "with_agent_purchases_usd": 0.038,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "5 new house listings in Austin 78745",
      "task_id": "cap-homes-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.8400000000000001,
      "with_completion": 0.9199999999999999,
      "completion_change_pp": 8.000000000000007,
      "without_cost_usd": 0.1199896,
      "with_cost_usd": 0.152679,
      "extra_cost_usd": 0.03268940000000001,
      "with_agent_purchases_usd": 0.006,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.89998,
      "with_completion": 1,
      "completion_change_pp": 10.001999999999999,
      "without_cost_usd": 0.1118562,
      "with_cost_usd": 0.108267,
      "extra_cost_usd": -0.0035892000000000007,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.83332,
      "with_completion": 0.9333199999999999,
      "completion_change_pp": 9.999999999999998,
      "without_cost_usd": 0.0582156,
      "with_cost_usd": 0.088939,
      "extra_cost_usd": 0.030723400000000005,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.66664,
      "with_completion": 0.8666600000000001,
      "completion_change_pp": 20.002,
      "without_cost_usd": 0.0046084,
      "with_cost_usd": 0.0043288,
      "extra_cost_usd": -0.0002796000000000005,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.83332,
      "with_completion": 0.8666600000000001,
      "completion_change_pp": 3.3339999999999925,
      "without_cost_usd": 0.08459219999999999,
      "with_cost_usd": 0.0861594,
      "extra_cost_usd": 0.0015672000000000047,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.63332,
      "with_completion": 0.6,
      "completion_change_pp": -3.3320000000000016,
      "without_cost_usd": 0.0053068,
      "with_cost_usd": 0.0028761999999999998,
      "extra_cost_usd": -0.0024306000000000006,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0.86664,
      "with_completion": 0.89998,
      "completion_change_pp": 3.3339999999999925,
      "without_cost_usd": 0.0871566,
      "with_cost_usd": 0.0797238,
      "extra_cost_usd": -0.007432800000000003,
      "with_agent_purchases_usd": 0.0004,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Fintech Series A market map",
      "task_id": "cap-market-map-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.9333199999999999,
      "with_completion": 0.73332,
      "completion_change_pp": -20.000000000000007,
      "without_cost_usd": 0.036245,
      "with_cost_usd": 0.055936200000000005,
      "extra_cost_usd": 0.019691200000000006,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.9,
      "with_completion": 1,
      "completion_change_pp": 9.999999999999998,
      "without_cost_usd": 0.40181940000000005,
      "with_cost_usd": 0.1726408,
      "extra_cost_usd": -0.22917860000000004,
      "with_agent_purchases_usd": 0.0124,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.9,
      "with_completion": 1,
      "completion_change_pp": 9.999999999999998,
      "without_cost_usd": 0.1887854,
      "with_cost_usd": 0.08478240000000001,
      "extra_cost_usd": -0.10400299999999998,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.5,
      "with_completion": 0.62,
      "completion_change_pp": 12,
      "without_cost_usd": 0.0094744,
      "with_cost_usd": 0.0135966,
      "extra_cost_usd": 0.004122200000000001,
      "with_agent_purchases_usd": 0.002,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.9,
      "with_completion": 0.9800000000000001,
      "completion_change_pp": 7.9999999999999964,
      "without_cost_usd": 0.9213546000000001,
      "with_cost_usd": 0.2557916,
      "extra_cost_usd": -0.6655630000000001,
      "with_agent_purchases_usd": 0.01,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.33999999999999997,
      "with_completion": 0.7,
      "completion_change_pp": 36,
      "without_cost_usd": 0.0385374,
      "with_cost_usd": 0.021501600000000003,
      "extra_cost_usd": -0.017035799999999997,
      "with_agent_purchases_usd": 0.004,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.1805508,
      "with_cost_usd": 0.13905019999999998,
      "extra_cost_usd": -0.041500600000000026,
      "with_agent_purchases_usd": 0.051000000000000004,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Meeting prep: CEO briefing*",
      "task_id": "cap-meeting-prep-v2",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.6,
      "with_completion": 0.6,
      "completion_change_pp": 0,
      "without_cost_usd": 0.141515,
      "with_cost_usd": 0.1263014,
      "extra_cost_usd": -0.015213599999999994,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.86876,
      "with_completion": 0.8187599999999999,
      "completion_change_pp": -4.999999999999993,
      "without_cost_usd": 0.3847502,
      "with_cost_usd": 0.8725052,
      "extra_cost_usd": 0.487755,
      "with_agent_purchases_usd": 0.089,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.7500199999999999,
      "with_completion": 0.93126,
      "completion_change_pp": 18.123999999999995,
      "without_cost_usd": 0.26107800000000003,
      "with_cost_usd": 0.49318419999999996,
      "extra_cost_usd": 0.23210619999999993,
      "with_agent_purchases_usd": 0.05,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.85002,
      "with_completion": 0.8562799999999999,
      "completion_change_pp": 0.6260000000000043,
      "without_cost_usd": 0.0163244,
      "with_cost_usd": 0.0376304,
      "extra_cost_usd": 0.021306000000000002,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.67504,
      "with_completion": 0.875,
      "completion_change_pp": 19.996000000000002,
      "without_cost_usd": 0.4392806,
      "with_cost_usd": 0.5963128,
      "extra_cost_usd": 0.15703219999999996,
      "with_agent_purchases_usd": 0.030000000000000006,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.8375199999999999,
      "with_completion": 0.8749800000000001,
      "completion_change_pp": 3.745999999999994,
      "without_cost_usd": 0.0165446,
      "with_cost_usd": 0.05050400000000001,
      "extra_cost_usd": 0.03395940000000001,
      "with_agent_purchases_usd": 0.032,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0.68126,
      "with_completion": 0.90622,
      "completion_change_pp": 22.496000000000006,
      "without_cost_usd": 0.15589840000000002,
      "with_cost_usd": 0.28532579999999996,
      "extra_cost_usd": 0.12942739999999994,
      "with_agent_purchases_usd": 0.0358,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Research 10 prospects, rank them, email the top 3",
      "task_id": "cap-prospects-v2",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.7562599999999999,
      "with_completion": 0.86252,
      "completion_change_pp": 10.62599999999999,
      "without_cost_usd": 0.266638,
      "with_cost_usd": 0.1294458,
      "extra_cost_usd": -0.1371922,
      "with_agent_purchases_usd": 0.02,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0,
      "with_completion": 0.9666600000000001,
      "completion_change_pp": 96.666,
      "without_cost_usd": 0.6080964,
      "with_cost_usd": 0.4932964,
      "extra_cost_usd": -0.11479999999999996,
      "with_agent_purchases_usd": 0.124,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0,
      "with_completion": 0.83332,
      "completion_change_pp": 83.33200000000001,
      "without_cost_usd": 0.1377042,
      "with_cost_usd": 0.3182314,
      "extra_cost_usd": 0.1805272,
      "with_agent_purchases_usd": 0.14,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 0,
      "completion_change_pp": 0,
      "without_cost_usd": 0.010866200000000001,
      "with_cost_usd": 0.0064358,
      "extra_cost_usd": -0.004430400000000001,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0,
      "with_completion": 0.73334,
      "completion_change_pp": 73.334,
      "without_cost_usd": 0.7793756000000001,
      "with_cost_usd": 0.429551,
      "extra_cost_usd": -0.34982460000000004,
      "with_agent_purchases_usd": 0.152,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 0,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0227826,
      "with_cost_usd": 0.025304399999999998,
      "extra_cost_usd": 0.0025217999999999977,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 0.93334,
      "completion_change_pp": 93.33399999999999,
      "without_cost_usd": 0.0763242,
      "with_cost_usd": 0.2591938,
      "extra_cost_usd": 0.18286959999999997,
      "with_agent_purchases_usd": 0.152,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "What parents on Reddit say about a car seat",
      "task_id": "cap-reddit-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0,
      "with_completion": 0.4,
      "completion_change_pp": 40,
      "without_cost_usd": 0.13776680000000002,
      "with_cost_usd": 0.18524279999999999,
      "extra_cost_usd": 0.04747599999999996,
      "with_agent_purchases_usd": 0.04,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.5466864,
      "with_cost_usd": 0.7259506,
      "extra_cost_usd": 0.17926419999999998,
      "with_agent_purchases_usd": 0.6,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.2738704,
      "with_cost_usd": 0.573844,
      "extra_cost_usd": 0.2999736,
      "with_agent_purchases_usd": 0.48,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0115782,
      "with_cost_usd": 0.0113544,
      "extra_cost_usd": -0.0002237999999999997,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.33632680000000004,
      "with_cost_usd": 0.7094028,
      "extra_cost_usd": 0.37307599999999996,
      "with_agent_purchases_usd": 0.61,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.011441,
      "with_cost_usd": 0.13395680000000001,
      "extra_cost_usd": 0.12251580000000001,
      "with_agent_purchases_usd": 0.122,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.17200359999999998,
      "with_cost_usd": 0.6292278,
      "extra_cost_usd": 0.4572242,
      "with_agent_purchases_usd": 0.6,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "OFAC sanctions screen for 12 counterparties",
      "task_id": "cap-sanctions-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.9833399999999999,
      "with_completion": 1,
      "completion_change_pp": 1.6660000000000008,
      "without_cost_usd": 0.09382660000000001,
      "with_cost_usd": 0.6800908,
      "extra_cost_usd": 0.5862642,
      "with_agent_purchases_usd": 0.4,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.2,
      "with_completion": 1,
      "completion_change_pp": 80,
      "without_cost_usd": 0.09188539999999999,
      "with_cost_usd": 0.21472660000000002,
      "extra_cost_usd": 0.12284120000000003,
      "with_agent_purchases_usd": 0.03,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0.31332,
      "with_completion": 1,
      "completion_change_pp": 68.66799999999999,
      "without_cost_usd": 0.2352766,
      "with_cost_usd": 0.1284724,
      "extra_cost_usd": -0.10680420000000002,
      "with_agent_purchases_usd": 0.038,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 0.64666,
      "completion_change_pp": 64.666,
      "without_cost_usd": 0.0026564,
      "with_cost_usd": 0.0993866,
      "extra_cost_usd": 0.0967302,
      "with_agent_purchases_usd": 0.092,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.38666,
      "with_completion": 1,
      "completion_change_pp": 61.334,
      "without_cost_usd": 0.3471222,
      "with_cost_usd": 0.2102462,
      "extra_cost_usd": -0.136876,
      "with_agent_purchases_usd": 0.054000000000000006,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.37333999999999995,
      "with_completion": 0.2,
      "completion_change_pp": -17.334,
      "without_cost_usd": 0.009901,
      "with_cost_usd": 0.1204086,
      "extra_cost_usd": 0.11050760000000001,
      "with_agent_purchases_usd": 0.11800000000000002,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.020760400000000002,
      "with_cost_usd": 0.1229242,
      "extra_cost_usd": 0.1021638,
      "with_agent_purchases_usd": 0.062,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Follower counts for 30 X accounts, from a CRM export with user IDs",
      "task_id": "cap-followers-batch-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.13334,
      "with_completion": 1,
      "completion_change_pp": 86.666,
      "without_cost_usd": 0.3543348,
      "with_cost_usd": 0.2004436,
      "extra_cost_usd": -0.1538912,
      "with_agent_purchases_usd": 0.10899999999999999,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.052088199999999994,
      "with_cost_usd": 0.043830400000000005,
      "extra_cost_usd": -0.008257799999999989,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.025731999999999998,
      "with_cost_usd": 0.036428,
      "extra_cost_usd": 0.010696000000000004,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0.52,
      "with_completion": 0.8400000000000001,
      "completion_change_pp": 31.999999999999996,
      "without_cost_usd": 0.0074676,
      "with_cost_usd": 0.005584400000000001,
      "extra_cost_usd": -0.001883199999999999,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0603838,
      "with_cost_usd": 0.0814024,
      "extra_cost_usd": 0.0210186,
      "with_agent_purchases_usd": 0.002,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 1,
      "with_completion": 0.8,
      "completion_change_pp": -19.999999999999996,
      "without_cost_usd": 0.0018017999999999999,
      "with_cost_usd": 0.3850508,
      "extra_cost_usd": 0.383249,
      "with_agent_purchases_usd": 0.382,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0174038,
      "with_cost_usd": 0.1944062,
      "extra_cost_usd": 0.1770024,
      "with_agent_purchases_usd": 0.172,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Five available .com names",
      "task_id": "cap-domains-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.6,
      "with_completion": 0.6,
      "completion_change_pp": 0,
      "without_cost_usd": 0.184669,
      "with_cost_usd": 0.430883,
      "extra_cost_usd": 0.24621400000000002,
      "with_agent_purchases_usd": 0.31779999999999997,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.006353,
      "with_cost_usd": 0.005779,
      "extra_cost_usd": -0.0005739999999999999,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.002439,
      "with_cost_usd": 0.002653,
      "extra_cost_usd": 0.00021399999999999978,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0001988,
      "with_cost_usd": 0.00020539999999999998,
      "extra_cost_usd": 0.000006599999999999971,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0035393999999999994,
      "with_cost_usd": 0.0119464,
      "extra_cost_usd": 0.008407,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.000151,
      "with_cost_usd": 0.0003404,
      "extra_cost_usd": 0.00018939999999999997,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0027052,
      "with_cost_usd": 0.003762,
      "extra_cost_usd": 0.0010568,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Next Friday's Apple closing price",
      "task_id": "cap-impossible-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0392318,
      "with_cost_usd": 0.09288919999999999,
      "extra_cost_usd": 0.053657399999999994,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.6,
      "with_completion": 0.5599999999999999,
      "completion_change_pp": -4.0000000000000036,
      "without_cost_usd": 0.090887,
      "with_cost_usd": 0.12259020000000001,
      "extra_cost_usd": 0.031703200000000015,
      "with_agent_purchases_usd": 0.009,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.015195,
      "with_cost_usd": 0.008139799999999999,
      "extra_cost_usd": -0.007055200000000001,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.003946,
      "with_cost_usd": 0.0022862,
      "extra_cost_usd": -0.0016598,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0.4,
      "with_completion": 0.4,
      "completion_change_pp": 0,
      "without_cost_usd": 0.057840199999999994,
      "with_cost_usd": 0.0700474,
      "extra_cost_usd": 0.012207200000000001,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.8,
      "with_completion": 0.2,
      "completion_change_pp": -60.00000000000001,
      "without_cost_usd": 0.004843,
      "with_cost_usd": 0.005716,
      "extra_cost_usd": 0.0008729999999999996,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0182352,
      "with_cost_usd": 0.0240948,
      "extra_cost_usd": 0.0058595999999999995,
      "with_agent_purchases_usd": 0.007400000000000001,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Last closing prices for 5 US stocks",
      "task_id": "cap-stocks-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0.8,
      "with_completion": 1,
      "completion_change_pp": 19.999999999999996,
      "without_cost_usd": 0.0462652,
      "with_cost_usd": 0.05140019999999999,
      "extra_cost_usd": 0.005134999999999994,
      "with_agent_purchases_usd": 0.0036,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0242372,
      "with_cost_usd": 0.042435,
      "extra_cost_usd": 0.0181978,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0239638,
      "with_cost_usd": 0.0180828,
      "extra_cost_usd": -0.005881000000000001,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0015772000000000002,
      "with_cost_usd": 0.0011322,
      "extra_cost_usd": -0.0004450000000000001,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0365502,
      "with_cost_usd": 0.0334746,
      "extra_cost_usd": -0.0030755999999999978,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0.4,
      "with_completion": 0.3,
      "completion_change_pp": -10.000000000000004,
      "without_cost_usd": 0.0023284,
      "with_cost_usd": 0.030603399999999996,
      "extra_cost_usd": 0.028274999999999995,
      "with_agent_purchases_usd": 0.024,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.007813200000000001,
      "with_cost_usd": 0.0240622,
      "extra_cost_usd": 0.016249,
      "with_agent_purchases_usd": 0.004,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Tomorrow's weather in Chicago",
      "task_id": "cap-weather-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 1,
      "with_completion": 0.7,
      "completion_change_pp": -30.000000000000004,
      "without_cost_usd": 0.050532600000000004,
      "with_cost_usd": 0.0439122,
      "extra_cost_usd": -0.0066204000000000054,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.008593,
      "with_cost_usd": 0.0111328,
      "extra_cost_usd": 0.0025398,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.004326999999999999,
      "with_cost_usd": 0.0189832,
      "extra_cost_usd": 0.0146562,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0016982,
      "with_cost_usd": 0.0011646,
      "extra_cost_usd": -0.0005336,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.028948800000000004,
      "with_cost_usd": 0.0508216,
      "extra_cost_usd": 0.021872799999999998,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 1,
      "with_completion": 0.8,
      "completion_change_pp": -19.999999999999996,
      "without_cost_usd": 0.0005098,
      "with_cost_usd": 0.041547,
      "extra_cost_usd": 0.0410372,
      "with_agent_purchases_usd": 0.04,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.004086,
      "with_cost_usd": 0.025641200000000003,
      "extra_cost_usd": 0.021555200000000004,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Apple.com visits yesterday",
      "task_id": "cap-traffic-v1",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 1,
      "with_completion": 1,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0227516,
      "with_cost_usd": 0.032767000000000004,
      "extra_cost_usd": 0.010015400000000004,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "Claude Opus 5.5",
      "model_id": "claude-opus-5-5",
      "without_completion": 0.7,
      "with_completion": 1,
      "completion_change_pp": 30.000000000000004,
      "without_cost_usd": 0.138716,
      "with_cost_usd": 0.0816414,
      "extra_cost_usd": -0.0570746,
      "with_agent_purchases_usd": 0.011399999999999999,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "Claude Sonnet 5.5",
      "model_id": "claude-sonnet-5-5",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.008625,
      "with_cost_usd": 0.0473136,
      "extra_cost_usd": 0.0386886,
      "with_agent_purchases_usd": 0.011199999999999998,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "Claude Haiku 5.5",
      "model_id": "claude-haiku-5-5",
      "without_completion": 0,
      "with_completion": 0,
      "completion_change_pp": 0,
      "without_cost_usd": 0.0006424,
      "with_cost_usd": 0.0014399999999999999,
      "extra_cost_usd": 0.0007975999999999999,
      "with_agent_purchases_usd": 0,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "Grok 4.6",
      "model_id": "grok-4.6",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.43948,
      "with_cost_usd": 0.23058579999999998,
      "extra_cost_usd": -0.2088942,
      "with_agent_purchases_usd": 0.012199999999999999,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "GPT-6 Luna",
      "model_id": "gpt-6-luna",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.008893,
      "with_cost_usd": 0.015068000000000002,
      "extra_cost_usd": 0.006175000000000002,
      "with_agent_purchases_usd": 0.0132,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "GPT-6.1 Sol",
      "model_id": "gpt-6.1-sol",
      "without_completion": 0.2,
      "with_completion": 1,
      "completion_change_pp": 80,
      "without_cost_usd": 0.0531992,
      "with_cost_usd": 0.031512200000000004,
      "extra_cost_usd": -0.021686999999999998,
      "with_agent_purchases_usd": 0.011,
      "without_runs": 5,
      "with_runs": 5
    },
    {
      "task": "Primes with digit sum 40",
      "task_id": "cap-compute-v2",
      "model": "Gemini 3 Flash",
      "model_id": "gemini-3-flash-preview",
      "without_completion": 0,
      "with_completion": 1,
      "completion_change_pp": 100,
      "without_cost_usd": 0.5020786,
      "with_cost_usd": 0.2112146,
      "extra_cost_usd": -0.290864,
      "with_agent_purchases_usd": 0.0262,
      "without_runs": 5,
      "with_runs": 5
    }
  ]
}