{
  "review_date": "2026-09-10",
  "source_url": "https://raw.githubusercontent.com/frontier-harness-eval/eval/e837a70bd6beb4e72eeeda62dd06e3bd34f6cb63/results/eval-data.json",
  "source_revision": "e837a70bd6beb4e72eeeda62dd06e3bd34f6cb63",
  "source_sha256": "ffd18213a165e985fd7d876395a3f53e00961253a89bbc88420a41374bc20c4c",
  "source_generated_at": "2026-08-22T16:04:57.538734+00:00",
  "method": "Audit of published outcomes; no agent trials reproduced",
  "cost_basis": "Sum of non-null cost_first_cold_usd / successes. Null costs remain missing, not zero; incomplete sums are lower bounds assuming nonnegative cost. Not verified invoices or total ownership cost.",
  "inference_limit": "Exploratory paired tests on a curated task sample, no multiplicity adjustment; not equivalence tests",
  "results": [
    {
      "configuration": "codex",
      "tasks": 30,
      "successes": 20,
      "pass_rate": 0.6666666666666666,
      "reported_first_cold_cost_usd": 69.3659265,
      "reported_cost_per_success_usd": 3.468296325,
      "cost_is_lower_bound": false,
      "tasks_with_reported_cost": 30,
      "tasks_with_missing_cost": [],
      "median_successful_task_first_cold_cost_usd": 0.12425444999999999
    },
    {
      "configuration": "dsh-creator",
      "tasks": 30,
      "successes": 19,
      "pass_rate": 0.6333333333333333,
      "reported_first_cold_cost_usd": 62.4131502,
      "reported_cost_per_success_usd": 3.284902642105263,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 27,
      "tasks_with_missing_cost": [
        "terminal-bench/largest-eigenval",
        "terminal-bench/kv-store-grpc",
        "terminal-bench/code-from-image"
      ],
      "median_successful_task_first_cold_cost_usd": 0.1194408
    },
    {
      "configuration": "claude-code",
      "tasks": 30,
      "successes": 19,
      "pass_rate": 0.6333333333333333,
      "reported_first_cold_cost_usd": 348.3998442,
      "reported_cost_per_success_usd": 18.33683390526316,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 29,
      "tasks_with_missing_cost": [
        "datacurve/anko-typed-variable-bindings"
      ],
      "median_successful_task_first_cold_cost_usd": 0.2879886
    },
    {
      "configuration": "pi-responses",
      "tasks": 30,
      "successes": 18,
      "pass_rate": 0.6,
      "reported_first_cold_cost_usd": 43.793379,
      "reported_cost_per_success_usd": 2.4329655,
      "cost_is_lower_bound": false,
      "tasks_with_reported_cost": 30,
      "tasks_with_missing_cost": [],
      "median_successful_task_first_cold_cost_usd": 0.07088475
    },
    {
      "configuration": "dsh-ptc",
      "tasks": 30,
      "successes": 18,
      "pass_rate": 0.6,
      "reported_first_cold_cost_usd": 82.39405289999999,
      "reported_cost_per_success_usd": 4.5774473833333325,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 27,
      "tasks_with_missing_cost": [
        "terminal-bench/largest-eigenval",
        "terminal-bench/kv-store-grpc",
        "terminal-bench/code-from-image"
      ],
      "median_successful_task_first_cold_cost_usd": 0.13697925
    },
    {
      "configuration": "dsh-standard",
      "tasks": 30,
      "successes": 18,
      "pass_rate": 0.6,
      "reported_first_cold_cost_usd": 62.259954,
      "reported_cost_per_success_usd": 3.4588863333333335,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 27,
      "tasks_with_missing_cost": [
        "terminal-bench/kv-store-grpc",
        "terminal-bench/chess-best-move",
        "datacurve/expr-try-catch-errors"
      ],
      "median_successful_task_first_cold_cost_usd": 0.12009539999999999
    },
    {
      "configuration": "oh-my-pi",
      "tasks": 30,
      "successes": 17,
      "pass_rate": 0.5666666666666667,
      "reported_first_cold_cost_usd": 80.674218,
      "reported_cost_per_success_usd": 4.745542235294117,
      "cost_is_lower_bound": false,
      "tasks_with_reported_cost": 30,
      "tasks_with_missing_cost": [],
      "median_successful_task_first_cold_cost_usd": 0.1353654
    },
    {
      "configuration": "kimi-code",
      "tasks": 30,
      "successes": 17,
      "pass_rate": 0.5666666666666667,
      "reported_first_cold_cost_usd": 62.0004801,
      "reported_cost_per_success_usd": 3.6470870647058824,
      "cost_is_lower_bound": false,
      "tasks_with_reported_cost": 30,
      "tasks_with_missing_cost": [],
      "median_successful_task_first_cold_cost_usd": 0.18175319999999998
    },
    {
      "configuration": "dsh-minimal",
      "tasks": 30,
      "successes": 17,
      "pass_rate": 0.5666666666666667,
      "reported_first_cold_cost_usd": 80.2052463,
      "reported_cost_per_success_usd": 4.717955664705882,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 26,
      "tasks_with_missing_cost": [
        "terminal-bench/largest-eigenval",
        "terminal-bench/kv-store-grpc",
        "terminal-bench/chess-best-move",
        "terminal-bench/code-from-image"
      ],
      "median_successful_task_first_cold_cost_usd": 0.12140220000000002
    },
    {
      "configuration": "exo",
      "tasks": 30,
      "successes": 16,
      "pass_rate": 0.5333333333333333,
      "reported_first_cold_cost_usd": 16.7227377,
      "reported_cost_per_success_usd": 1.04517110625,
      "cost_is_lower_bound": false,
      "tasks_with_reported_cost": 30,
      "tasks_with_missing_cost": [],
      "median_successful_task_first_cold_cost_usd": 0.07481280000000001
    },
    {
      "configuration": "opencode",
      "tasks": 30,
      "successes": 15,
      "pass_rate": 0.5,
      "reported_first_cold_cost_usd": 48.6651828,
      "reported_cost_per_success_usd": 3.24434552,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 29,
      "tasks_with_missing_cost": [
        "datacurve/python-statemachine-state-data-scoping"
      ],
      "median_successful_task_first_cold_cost_usd": 0.0614517
    },
    {
      "configuration": "hermes",
      "tasks": 30,
      "successes": 15,
      "pass_rate": 0.5,
      "reported_first_cold_cost_usd": 43.5645522,
      "reported_cost_per_success_usd": 2.9043034800000003,
      "cost_is_lower_bound": true,
      "tasks_with_reported_cost": 28,
      "tasks_with_missing_cost": [
        "datacurve/arktype-json-schema-refs-dependencies",
        "datacurve/expr-try-catch-errors"
      ],
      "median_successful_task_first_cold_cost_usd": 0.174564
    }
  ],
  "paired_comparisons": [
    {
      "baseline": "codex",
      "comparison": "dsh-creator",
      "baseline_only_successes": 4,
      "comparison_only_successes": 3,
      "exact_two_sided_p_unadjusted": 1.0
    },
    {
      "baseline": "codex",
      "comparison": "claude-code",
      "baseline_only_successes": 3,
      "comparison_only_successes": 2,
      "exact_two_sided_p_unadjusted": 1.0
    },
    {
      "baseline": "codex",
      "comparison": "pi-responses",
      "baseline_only_successes": 4,
      "comparison_only_successes": 2,
      "exact_two_sided_p_unadjusted": 0.6875
    },
    {
      "baseline": "codex",
      "comparison": "dsh-ptc",
      "baseline_only_successes": 4,
      "comparison_only_successes": 2,
      "exact_two_sided_p_unadjusted": 0.6875
    },
    {
      "baseline": "codex",
      "comparison": "dsh-standard",
      "baseline_only_successes": 5,
      "comparison_only_successes": 3,
      "exact_two_sided_p_unadjusted": 0.7265625
    },
    {
      "baseline": "codex",
      "comparison": "oh-my-pi",
      "baseline_only_successes": 5,
      "comparison_only_successes": 2,
      "exact_two_sided_p_unadjusted": 0.453125
    },
    {
      "baseline": "codex",
      "comparison": "kimi-code",
      "baseline_only_successes": 5,
      "comparison_only_successes": 2,
      "exact_two_sided_p_unadjusted": 0.453125
    },
    {
      "baseline": "codex",
      "comparison": "dsh-minimal",
      "baseline_only_successes": 4,
      "comparison_only_successes": 1,
      "exact_two_sided_p_unadjusted": 0.375
    },
    {
      "baseline": "codex",
      "comparison": "exo",
      "baseline_only_successes": 6,
      "comparison_only_successes": 2,
      "exact_two_sided_p_unadjusted": 0.2890625
    },
    {
      "baseline": "codex",
      "comparison": "opencode",
      "baseline_only_successes": 6,
      "comparison_only_successes": 1,
      "exact_two_sided_p_unadjusted": 0.125
    },
    {
      "baseline": "codex",
      "comparison": "hermes",
      "baseline_only_successes": 5,
      "comparison_only_successes": 0,
      "exact_two_sided_p_unadjusted": 0.0625
    }
  ]
}