"""Recompute the September 10, 2026 article from a pinned public JSON file.

Python 3.10+, standard library only. No model calls or network requests.
Usage: python3 frontierharness_audit.py /path/to/eval-data.json
The source URL and SHA-256 are recorded in the output. This is a data audit,
not a rerun of agent trials. Pairwise p-values are exploratory and unadjusted.
"""

import hashlib
import json
import math
from pathlib import Path
import statistics
import sys

REVISION = "e837a70bd6beb4e72eeeda62dd06e3bd34f6cb63"
SOURCE = (
    "https://raw.githubusercontent.com/frontier-harness-eval/eval/"
    + REVISION + "/results/eval-data.json"
)
EXPECTED_SHA256 = "ffd18213a165e985fd7d876395a3f53e00961253a89bbc88420a41374bc20c4c"


def exact_paired_p(wins, losses):
    """Two-sided exact McNemar test, conditional on discordant pairs."""
    n = wins + losses
    if n == 0:
        return 1.0
    tail = sum(math.comb(n, k) for k in range(min(wins, losses) + 1))
    return min(1.0, 2 * tail / 2**n)


def audit(raw):
    digest = hashlib.sha256(raw).hexdigest()
    if digest != EXPECTED_SHA256:
        raise ValueError("Source hash differs from the article's pinned dataset")
    data = json.loads(raw)
    rows = []
    outcomes = {}
    for harness in data["harnesses"]:
        tasks = harness["task_details"]
        ids = [task["id"] for task in tasks]
        if len(tasks) != 30 or len(set(ids)) != 30:
            raise ValueError("Expected 30 unique task results per configuration")
        if any(type(t["success"]) is not bool for t in tasks):
            raise ValueError("Expected explicit boolean success values")
        missing_cost_ids = [t["id"] for t in tasks if t["cost_first_cold_usd"] is None]
        costs = [t["cost_first_cold_usd"] for t in tasks if t["cost_first_cold_usd"] is not None]
        if any(type(c) not in (int, float) or not math.isfinite(c) or c < 0 for c in costs):
            raise ValueError("Missing, invalid, or negative cost")
        successful = [t for t in tasks if t["success"]]
        if any(t["cost_first_cold_usd"] is None for t in successful):
            raise ValueError("Successful-task median requires complete successful-task costs")
        if not successful:
            raise ValueError("Cost per success is undefined with zero successes")
        effective = math.fsum(costs) / len(successful)
        if not math.isclose(effective, harness["effective_cost_per_pass"], rel_tol=1e-12):
            raise ValueError("Recomputed cost does not match aggregate field")
        rows.append({
            "configuration": harness["name"],
            "tasks": len(tasks),
            "successes": len(successful),
            "pass_rate": len(successful) / len(tasks),
            "reported_first_cold_cost_usd": math.fsum(costs),
            "reported_cost_per_success_usd": effective,
            "cost_is_lower_bound": bool(missing_cost_ids),
            "tasks_with_reported_cost": len(costs),
            "tasks_with_missing_cost": missing_cost_ids,
            "median_successful_task_first_cold_cost_usd": statistics.median(
                t["cost_first_cold_usd"] for t in successful
            ),
        })
        outcomes[harness["name"]] = dict(zip(ids, (t["success"] for t in tasks)))
    baseline = outcomes["codex"]
    pairs = []
    for name, result in outcomes.items():
        if set(result) != set(baseline):
            raise ValueError("Unmatched task identities")
        if name == "codex":
            continue
        wins = sum(baseline[k] and not result[k] for k in baseline)
        losses = sum(not baseline[k] and result[k] for k in baseline)
        pairs.append({
            "baseline": "codex", "comparison": name,
            "baseline_only_successes": wins,
            "comparison_only_successes": losses,
            "exact_two_sided_p_unadjusted": exact_paired_p(wins, losses),
        })
    return {
        "review_date": "2026-09-10",
        "source_url": SOURCE, "source_revision": REVISION,
        "source_sha256": digest,
        "source_generated_at": data["generated_at"],
        "method": "Audit of published outcomes; no agent trials reproduced",
        "cost_basis": "Sum of non-null cost_first_cold_usd / successes. Null costs remain missing, not zero; incomplete sums are lower bounds assuming nonnegative cost. Not verified invoices or total ownership cost.",
        "inference_limit": "Exploratory paired tests on a curated task sample, no multiplicity adjustment; not equivalence tests",
        "results": rows, "paired_comparisons": pairs,
    }


if __name__ == "__main__":
    if len(sys.argv) != 2:
        raise SystemExit("Usage: python3 frontierharness_audit.py eval-data.json")
    print(json.dumps(audit(Path(sys.argv[1]).read_bytes()), indent=2, allow_nan=False))