From 864f6a5caa236dc641897da1b651dd65c85691b6 Mon Sep 17 00:00:00 2001 From: ipezygj Date: Mon, 17 Aug 2026 17:28:30 +0300 Subject: [PATCH 1/4] Leaderboard: sampling error and McNemar significance groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The board reports a resolve rate per entry and nothing about how precisely it is measured, so adjacent rows read as ordered even when the data does not order them. This adds two columns computed from per-instance results the site already carries: ± SE binomial standard error of that entry's own rate, in percentage points, over the instances it was scored on. Stated as the precision of one number, not as the error of a difference: entries share an instance set, so comparing two of them is a paired question. Tie that paired question, answered. Exact two-sided McNemar at alpha = 0.05; the highest entry that publishes per-instance results anchors group 1 and every entry the test cannot separate from it joins the group. Membership is a property of the comparison with the anchor, not of an entry alone. Nothing is stored: build.py annotates the loaded leaderboards, so leaderboards.json stays the record of what was submitted and these stay a function of it. Entries without per-instance results get a dash rather than a blank, and sort last in both directions, so "not published" cannot read as a small error or a top group. The columns appear when the group anchor is the entry at the top of the view, which today means the bash-only view. On the current data this puts the top six bash-only entries (76.8 down to 74.2) in one group. leaderboard_statistics.py --selftest covers the test, the SE, the anchor rule and the empty cases. --- build.py | 21 +++ css/leaderboard.css | 39 +++++ data/leaderboard_statistics.py | 278 +++++++++++++++++++++++++++++++++ js/mainResults.js | 59 ++++++- 4 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 data/leaderboard_statistics.py diff --git a/build.py b/build.py index 6127e4d..d118800 100644 --- a/build.py +++ b/build.py @@ -2,6 +2,7 @@ import json import pathlib import shutil +import sys from flask import Flask from flask_flatpages import FlatPages from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -11,6 +12,9 @@ TEMPLATES = ROOT / "templates" DIST = ROOT / "dist" +sys.path.insert(0, str(ROOT / "data")) +from leaderboard_statistics import annotate_all # noqa: E402 + def get_pages(): pages = {} pages_dir = TEMPLATES / "pages" @@ -126,6 +130,23 @@ def main() -> None: # load data with open(ROOT / "data/leaderboards.json", "r") as f: leaderboards = json.load(f) + + # Sampling error and significance groups, computed here rather than stored, + # so that leaderboards.json stays the record of what was submitted and these + # stay a function of it. Boards without per-instance results are untouched. + for summary in annotate_all( + leaderboards["leaderboards"] if isinstance(leaderboards, dict) else leaderboards + ): + line = (f"stats: {summary['name']}: {summary['with_per_instance']}/{summary['entries']} " + f"entries with per-instance results") + if summary["groups"]: + line += (f", {summary['groups']} significance groups, " + f"{summary['top_tie_group_size']} in group 1 anchored on " + f"{summary['anchor']}") + if summary["unmeasured_above_anchor"]: + line += f" ({summary['unmeasured_above_anchor']} higher entries publish none)" + print(line) + with open(ROOT / "data/press.json", "r") as f: press = json.load(f) press = sorted(press, key=lambda x: x["date"], reverse=True) diff --git a/css/leaderboard.css b/css/leaderboard.css index 90edebb..e8bddfe 100644 --- a/css/leaderboard.css +++ b/css/leaderboard.css @@ -782,10 +782,49 @@ min-width: 66rem; } +/* The two statistics columns add sized width, so the point at which the wrapper + starts scrolling instead of squeezing Model moves with them. */ +#leaderboard-container .data-table[data-stats="true"] { + min-width: 74rem; +} + #leaderboard-container col.cw-select { width: 2rem; } #leaderboard-container col.cw-rank { width: 3.1rem; } #leaderboard-container col.cw-agent { width: 12rem; } #leaderboard-container col.cw-resolved { width: 7.25rem; } +/* Statistics columns: the error on one entry, and the paired group it sits in */ +#leaderboard-container .stat-cell .number, +#leaderboard-container .stat-cell .text-muted { + font-variant-numeric: tabular-nums; +} + +#leaderboard-container .tie-group { + display: inline-block; + min-width: 1.5rem; + padding: 0.05rem 0.35rem; + border-radius: var(--radius-sm, 0.25rem); + font-variant-numeric: tabular-nums; + color: var(--color-text-muted); +} + +/* Group 1 is the one the headline number is read off, so it is the one worth + seeing at a glance. Emphasis, not a claim of quality. The tint is the same + accent the site already defines for both themes, so it stays visible in dark + mode instead of disappearing into the background. */ +#leaderboard-container .tie-group-top { + background-color: var(--color-accent-light); + color: var(--color-text); + font-weight: 600; +} + +.stats-note { + margin: 0.5rem 0 0; + font-size: 0.8125rem; + line-height: 1.5; +} + +#leaderboard-container col.cw-se { width: 4.75rem; } +#leaderboard-container col.cw-tie { width: 3.5rem; } #leaderboard-container col.cw-cost { width: 5.25rem; } #leaderboard-container col.cw-trajs { width: 4.5rem; } #leaderboard-container col.cw-org { width: 4.75rem; } diff --git a/data/leaderboard_statistics.py b/data/leaderboard_statistics.py new file mode 100644 index 0000000..b1b9a16 --- /dev/null +++ b/data/leaderboard_statistics.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Sampling error and significance groups for leaderboard entries. + +Two numbers are added to every entry that publishes per-instance results: + + resolved_se the binomial standard error of THAT entry's resolve rate over + the instances it was scored on, in percentage points: + 100 * sqrt(p*(1-p)/n). It is the sampling error of one rate on + a fixed instance set. It is NOT the error of a difference + between two entries: two entries are scored on the same + instances, so their difference is a paired quantity with a + smaller error than these two marginals suggest. Read this + column as "how precisely is this one number measured", never + as "these two entries overlap, therefore they are tied". + + tie_group the significance group, from the paired comparison that the SE + column deliberately does not perform. Entries are ordered by + resolve rate; the highest ungrouped entry anchors a group, and + every entry an exact two-sided McNemar test cannot separate + from that anchor at alpha = 0.05 joins it. Group membership is + therefore a statement about the comparison with the anchor, + not a claim that all members are mutually indistinguishable, + and not a property of any entry on its own. + +Both require per-instance outcomes. Entries without them get neither number, and +the caller is expected to render that as "not published" rather than as a blank +that reads like a pass. Where the highest entry on a board publishes nothing, the +anchor is the highest entry that does, and the summary says who it is and how +many unmeasured entries rank above it — a group is a statement about a comparison, +and it cannot be read without knowing what the comparison was against. + +No multiplicity correction is applied to the comparisons against an anchor. A +correction would make separation harder and the groups larger, so the group +sizes produced here are a lower bound on how much of a board is statistically +indistinguishable. + + python leaderboard_statistics.py --selftest + python leaderboard_statistics.py leaderboards.json # report, writes nothing +""" + +from __future__ import annotations + +import argparse +import json +import math +import pathlib +import sys + +ALPHA = 0.05 + + +def _binom_cdf_half(k: int, n: int) -> float: + """P(X <= k) for X ~ Binomial(n, 0.5), computed exactly.""" + return sum(math.comb(n, i) for i in range(k + 1)) / (2.0**n) + + +def mcnemar_exact(a: dict, b: dict) -> float: + """Two-sided exact McNemar p-value over the instances both entries report. + + a and b map instance id -> bool (resolved). Only shared instances are used, + because the test is paired: an instance one entry never attempted carries no + information about the difference between them. + """ + shared = a.keys() & b.keys() + a_only = sum(1 for i in shared if a[i] and not b[i]) + b_only = sum(1 for i in shared if b[i] and not a[i]) + n = a_only + b_only + if n == 0: + return 1.0 + return min(1.0, 2.0 * _binom_cdf_half(min(a_only, b_only), n)) + + +def binomial_se_pp(resolved: int, n: int) -> float: + """Standard error of a resolve rate, in percentage points.""" + if n <= 0: + return 0.0 + p = resolved / n + return 100.0 * math.sqrt(p * (1.0 - p) / n) + + +def _outcomes(entry: dict) -> dict | None: + details = entry.get("per_instance_details") + if not details: + return None + return {k: bool(v.get("resolved")) for k, v in details.items()} + + +def annotate(leaderboard: dict, alpha: float = ALPHA) -> dict: + """Add resolved_se / n_instances / tie_group in place. Returns a summary.""" + results = leaderboard.get("results", []) + ranked = sorted(results, key=lambda r: -(float(r.get("resolved") or 0))) + + for entry in results: + outcomes = _outcomes(entry) + if outcomes is None: + continue + n = len(outcomes) + entry["n_instances"] = n + entry["resolved_se"] = round(binomial_se_pp(sum(outcomes.values()), n), 2) + + measurable = [r for r in ranked if _outcomes(r) is not None] + summary = { + "name": leaderboard.get("name"), + "entries": len(results), + "with_per_instance": len(measurable), + "grouped": 0, + "groups": 0, + "top_tie_group_size": 0, + "anchor": None, + "unmeasured_above_anchor": 0, + } + if not measurable: + return summary + + # Group 1 is anchored on the highest entry that publishes per-instance + # results, which is not always the highest entry on the board. Whoever it is + # gets named, because a group is a statement about a comparison and the + # reader cannot check it without knowing what the comparison was against. + anchor_index = ranked.index(measurable[0]) + summary["anchor"] = measurable[0].get("model_display") or measurable[0].get("name") + summary["unmeasured_above_anchor"] = anchor_index + + pending = measurable + group = 0 + while pending: + group += 1 + anchor, rest = pending[0], pending[1:] + anchor_outcomes = _outcomes(anchor) + anchor["tie_group"] = group + pending = [] + for entry in rest: + if mcnemar_exact(anchor_outcomes, _outcomes(entry)) >= alpha: + entry["tie_group"] = group + else: + pending.append(entry) + if group == 1: + summary["top_tie_group_size"] = 1 + sum( + 1 for e in rest if e.get("tie_group") == 1 + ) + + summary["groups"] = group + summary["grouped"] = sum(1 for r in results if r.get("tie_group")) + leaderboard["statistics"] = dict(summary, alpha=alpha) + return summary + + +def annotate_all(leaderboards: list[dict], alpha: float = ALPHA) -> list[dict]: + return [annotate(lb, alpha) for lb in leaderboards] + + +# ------------------------------------------------------------------ selftest + +def _entry(name: str, outcomes: list[int]) -> dict: + return { + "name": name, + "resolved": 100.0 * sum(outcomes) / len(outcomes), + "per_instance_details": { + f"i{i}": {"resolved": bool(v)} for i, v in enumerate(outcomes) + }, + } + + +def selftest() -> int: + failures = [] + + # An entry compared with itself is never separated, at any size. + same = [i % 3 == 0 for i in range(500)] + a = {f"i{i}": v for i, v in enumerate(same)} + if mcnemar_exact(a, dict(a)) != 1.0: + failures.append("identical entries were separated") + + # The control above must not pass for lack of power: plant flips one way. + b = dict(a) + flipped = 0 + for k in list(b): + if b[k] and flipped < 40: + b[k] = False + flipped += 1 + if mcnemar_exact(a, b) >= 1e-6: + failures.append("40 planted one-directional flips were not detected") + + # Discordance that is balanced is not evidence of a difference, however big. + c, d = {}, {} + for i in range(200): + c[f"i{i}"], d[f"i{i}"] = (i % 2 == 0), (i % 2 == 1) + if mcnemar_exact(c, d) < ALPHA: + failures.append("200 balanced discordant pairs were called a difference") + + # SE is the textbook binomial one, and zero variance means zero error. + if abs(binomial_se_pp(250, 500) - 100 * math.sqrt(0.25 / 500)) > 1e-9: + failures.append("binomial SE does not match sqrt(p(1-p)/n)") + if binomial_se_pp(500, 500) != 0.0: + failures.append("a perfect score did not get a zero SE") + + # Grouping: two indistinguishable leaders and one clearly worse entry give + # group 1 = {leader, near-tie} and group 2 = {laggard}. + lead = [1] * 260 + [0] * 240 + near = lead[:] + near[0], near[1], near[259] = 0, 0, 0 + near[300], near[301] = 1, 1 + lag = [1] * 150 + [0] * 350 + lb = {"name": "t", "results": [_entry("lead", lead), _entry("near", near), _entry("lag", lag)]} + s = annotate(lb) + got = {r["name"]: r.get("tie_group") for r in lb["results"]} + if got != {"lead": 1, "near": 1, "lag": 2}: + failures.append(f"grouping was {got}, expected lead/near in 1 and lag in 2") + if s["top_tie_group_size"] != 2: + failures.append(f"top group size was {s['top_tie_group_size']}, expected 2") + + # An entry with no per-instance results gets no numbers and no group, and it + # does not shift anyone else's group. + lb2 = {"name": "t2", "results": [_entry("lead", lead), {"name": "silent", "resolved": 40.0}, + _entry("lag", lag)]} + annotate(lb2) + silent = lb2["results"][1] + if "tie_group" in silent or "resolved_se" in silent: + failures.append("an entry without per-instance results was given statistics") + if {r["name"]: r.get("tie_group") for r in lb2["results"] if "per_instance_details" in r} != { + "lead": 1, "lag": 2}: + failures.append("a silent entry changed the groups of the others") + + # A board whose top entry publishes nothing still gets groups, anchored on + # the highest entry that does — and that entry is named, with a count of how + # many unmeasured entries rank above it, so the anchor is never implicit. + lb3 = {"name": "t3", "results": [{"name": "top", "resolved": 99.0}, _entry("lead", lead), + _entry("lag", lag)]} + s3 = annotate(lb3) + if s3["anchor"] != "lead" or s3["unmeasured_above_anchor"] != 1: + failures.append(f"anchor was {s3['anchor']!r} with {s3['unmeasured_above_anchor']} " + "unmeasured above it, expected 'lead' with 1") + if lb3["results"][0].get("tie_group") is not None: + failures.append("an entry with no per-instance results was put in a group") + if s3["groups"] != 2: + failures.append(f"expected 2 groups under an unmeasured top entry, got {s3['groups']}") + + # A board where nothing is measurable produces no groups and no anchor. + lb4 = {"name": "t4", "results": [{"name": "a", "resolved": 9.0}, {"name": "b", "resolved": 8.0}]} + s4 = annotate(lb4) + if s4["groups"] or s4["anchor"] is not None or "statistics" in lb4: + failures.append("a board with no per-instance results was given statistics") + + # Only shared instances are compared. + short = {"i0": True, "i1": False} + long_ = {"i0": True, "i1": False, "i2": True, "i3": True} + if mcnemar_exact(short, long_) != 1.0: + failures.append("instances missing from one entry were treated as failures") + + if failures: + print("SELFTEST FAILED") + for f in failures: + print(" -", f) + return 1 + print("selftest passed: identical entries never separated, planted flips detected, " + "balanced discordance not called a difference, SE matches the binomial formula, " + "the anchor is the highest measurable entry and is named, missing per-instance " + "results stay empty") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("json", nargs="?", help="leaderboards.json to report on (not modified)") + ap.add_argument("--selftest", action="store_true") + args = ap.parse_args() + + if args.selftest or not args.json: + return selftest() + + data = json.loads(pathlib.Path(args.json).read_text(encoding="utf-8")) + boards = data["leaderboards"] if isinstance(data, dict) else data + for s in annotate_all(boards): + print(f"{s['name']:<14} entries={s['entries']:>4} with per-instance={s['with_per_instance']:>4} " + f"grouped={s['grouped']:>4} groups={s['groups']:>3} top group={s['top_tie_group_size']:>3}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/js/mainResults.js b/js/mainResults.js index a703a78..c9c4efa 100644 --- a/js/mainResults.js +++ b/js/mainResults.js @@ -5,6 +5,11 @@ let leaderboardData = null; const sortState = { field: 'resolved', direction: 'desc' }; +// A cell that has nothing in it is not a small value: it sorts last whichever +// way the arrow points, so that "no per-instance results published" never reads +// as "the smallest error" or "the top group". +const MISSING = Symbol('missing'); + function escapeAttr(value) { return String(value == null ? '' : value).replace(/[&<>"']/g, ch => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' @@ -46,6 +51,11 @@ function sortItems(a, b, field, direction) { return item.trajs_docent && item.trajs_docent !== false ? 1 : 0; case 'release': return (item['mini-swe-agent_version'] || '').toLowerCase(); + // Entries without per-instance results have no SE and no group. + case 'resolved_se': + return item.resolved_se == null ? MISSING : parseFloat(item.resolved_se); + case 'tie_group': + return item.tie_group == null ? MISSING : parseInt(item.tie_group, 10); default: return ''; } @@ -53,7 +63,12 @@ function sortItems(a, b, field, direction) { const av = getValue(a, field); const bv = getValue(b, field); - + + if (av === MISSING || bv === MISSING) { + if (av === MISSING && bv === MISSING) return 0; + return av === MISSING ? 1 : -1; + } + let result; if (typeof av === 'number' && typeof bv === 'number') { result = av - bv; @@ -204,7 +219,16 @@ function renderLeaderboardTable(leaderboard) { const displayNames = buildDisplayNames(results); const modelName = item => item.model_display || displayNames.get(item.name) || item.name; - const columnCount = 5 + (withSelect ? 1 : 0) + (withHarness ? 3 : 1); + // The two statistics columns need per-instance results, which only some + // submissions publish. They are shown when the group anchor is the row the + // reader is looking at the top of: either nothing on the board outranks it, + // or the view has been narrowed to the harness whose runs publish them. + // Anywhere else the column would be mostly dashes anchored on a row far + // down the page, which says less than leaving it out and explaining why. + const stats = leaderboard.statistics; + const withStats = !!stats && stats.groups > 0 + && (withHarness || stats.unmeasured_above_anchor === 0); + const columnCount = 5 + (withSelect ? 1 : 0) + (withHarness ? 3 : 1) + (withStats ? 2 : 0); // The table uses a fixed layout so the narrow columns keep their width no // matter how many other columns are on screen. Every column is sized here @@ -215,6 +239,8 @@ function renderLeaderboardTable(leaderboard) { '', '', '', + withStats ? '' : '', + withStats ? '' : '', withHarness ? '' : '', withHarness ? '' : '', '', @@ -225,7 +251,7 @@ function renderLeaderboardTable(leaderboard) { const tableHtml = `
- +
${colgroup} @@ -234,6 +260,8 @@ function renderLeaderboardTable(leaderboard) { + ${withStats ? '' : ''} + ${withStats ? '' : ''} ${withHarness ? '' : ''} ${withHarness ? '' : ''} @@ -274,6 +302,12 @@ function renderLeaderboardTable(leaderboard) { + ${withStats ? `` : ''} + ${withStats ? `` : ''} ${withHarness ? `` : ''} ${withHarness ? `
Model Agent % Resolved± SETieAvg. $TrajsOrg ${item.resolved_se != null + ? `±${parseFloat(item.resolved_se).toFixed(2)}` + : '—'}${item.tie_group != null + ? `${item.tie_group}` + : '—'}${item.instance_cost !== null && item.instance_cost !== undefined && item.instance_cost !== 0 && !isNaN(item.instance_cost) ? '$' + parseFloat(item.instance_cost).toFixed(2) : ''} ${item.trajs_docent && item.trajs_docent !== false ? `` : '-'} @@ -301,6 +335,25 @@ function renderLeaderboardTable(leaderboard) {
+ ${withStats ? ` +

+ ± SE is the standard error of a single entry's resolve rate, + sqrt(p(1-p)/n) in percentage points, over the + instances that entry was scored on. Two entries are scored on the same + instances, so whether they differ is a paired question and overlapping SEs are + not the answer to it. + Tie answers that paired question: entries an exact two-sided McNemar test + cannot separate (α = 0.05) from the group's anchor share its + number, group 1 being anchored on + ${stats ? escapeAttr(stats.anchor) : ''}, the highest entry here that + publishes per-instance results. It is a property of that comparison, not of an + entry on its own. No multiplicity correction is applied, so the groups are if + anything smaller than a corrected analysis would make them. + Computed over the ${stats ? stats.with_per_instance : 0} of + ${stats ? stats.entries : 0} entries on this board that publish per-instance + results. A dash means an entry publishes none, so neither number exists for + it — it does not mean the entry stands alone. +

` : ''}
`; From 6f74bfa7be98253242312186c12d69c97ed6e567 Mon Sep 17 00:00:00 2001 From: ipezygj Date: Wed, 19 Aug 2026 16:54:43 +0300 Subject: [PATCH 2/4] Say what neither column measures: repeatability Both numbers condition on the single run each entry reports. Nothing here bounds how much a score would move on a rerun of the same system, so neither column is the uncertainty of the ranking. Stated in the note a reader sees and in the module docstring, next to the two claims that were already explicit (SE is not the error of a difference; the group is a paired result against a named anchor at alpha = 0.05 with no multiplicity correction). Raised by @KeilerHirsch in SWE-bench/SWE-bench#621: marginal uncertainty, paired comparison resolution and repeatability are three different things and the wording should not let one be read as another. --- data/leaderboard_statistics.py | 5 +++++ js/mainResults.js | 3 +++ 2 files changed, 8 insertions(+) diff --git a/data/leaderboard_statistics.py b/data/leaderboard_statistics.py index b1b9a16..377cbca 100644 --- a/data/leaderboard_statistics.py +++ b/data/leaderboard_statistics.py @@ -34,6 +34,11 @@ sizes produced here are a lower bound on how much of a board is statistically indistinguishable. +Neither number is a repeatability estimate. Both condition on the single run each +entry reports. A rerun of the same system on the same instances is not observable +in this data, so nothing here bounds how much a score would move on a second run, +and neither column may be read as the uncertainty of the ranking itself. + python leaderboard_statistics.py --selftest python leaderboard_statistics.py leaderboards.json # report, writes nothing """ diff --git a/js/mainResults.js b/js/mainResults.js index c9c4efa..8a2659d 100644 --- a/js/mainResults.js +++ b/js/mainResults.js @@ -349,6 +349,9 @@ function renderLeaderboardTable(leaderboard) { publishes per-instance results. It is a property of that comparison, not of an entry on its own. No multiplicity correction is applied, so the groups are if anything smaller than a corrected analysis would make them. + Neither number is a repeatability estimate: both condition on the single run + each entry reports, and a rerun of the same system is not observable here, so + neither is the uncertainty of the ranking. Computed over the ${stats ? stats.with_per_instance : 0} of ${stats ? stats.entries : 0} entries on this board that publish per-instance results. A dash means an entry publishes none, so neither number exists for From cdf2052e804aa129e257ae0e1f58bee851e40cd3 Mon Sep 17 00:00:00 2001 From: ipezygj Date: Wed, 19 Aug 2026 19:41:52 +0300 Subject: [PATCH 3/4] Do not compute statistics from per-instance results that contradict the board Both columns are computed from per-instance outcomes while the board's own number comes from elsewhere, so the two can be checked against each other. They now are: an entry is annotated only when the rate implied by its per-instance results agrees with its published rate to within one instance (100/n percentage points). One instance is the finest step the published rate can move in, so a smaller gap is rounding or a denominator that differs by one, and a larger one means the two artifacts disagree about at least one instance. This is not hypothetical. On the current data two submissions publish per-instance files that imply 0.00 % and 10.20 % against published rates of 69.60 % and 52.80 %. Before this commit the first of them was rendered with the most precise-looking cell on the page, an SE of +/- 0.00, because a rate of exactly zero makes sqrt(p(1-p)/n) vanish; the second was placed in a significance group computed from a 10.20 % outcome set. Both now show a dash whose tooltip says which of the two published numbers disagrees with which, and the build prints them by name so the gap is not only visible on the page. The anchor and the size of the top group are unchanged on every board. Three further cases where a formula returns a number that is not a measurement: - an empty sample raises instead of returning an SE of 0.0; - a boundary rate (every instance resolved, or none) withholds the SE, since the zero is a property of the estimator rather than of the entry, while the entry still takes part in the paired comparisons, which remain valid; - two entries that share no instance are no longer given a p-value of 1.0 and put in the same group, which reported absence of paired evidence as absence of difference. mcnemar_exact returns None there and the entry anchors its own group. Selftest covers each, including both sides of the one-instance threshold, and each guard was checked by mutation: loosening the gate by 2x or 10x, tightening it 10x, returning 0.0 for an empty sample, treating disjoint sets as a tie, and publishing the boundary zero all fail the selftest. --- build.py | 8 ++ data/leaderboard_statistics.py | 168 ++++++++++++++++++++++++++++++--- js/mainResults.js | 25 ++++- 3 files changed, 187 insertions(+), 14 deletions(-) diff --git a/build.py b/build.py index d118800..de66e83 100644 --- a/build.py +++ b/build.py @@ -145,7 +145,15 @@ def main() -> None: f"{summary['anchor']}") if summary["unmeasured_above_anchor"]: line += f" ({summary['unmeasured_above_anchor']} higher entries publish none)" + if summary["not_comparable"]: + line += (f", {summary['not_comparable']} entries share no instance with the " + "anchor of their group") print(line) + # An entry dropped for contradicting its own published rate is a defect + # in the submitted data, not a quiet gap in a column, so the build says + # so by name rather than leaving a dash on the page as the only trace. + for name, reason in summary["excluded"]: + print(f"stats: {summary['name']}: {name}: {reason}") with open(ROOT / "data/press.json", "r") as f: press = json.load(f) diff --git a/data/leaderboard_statistics.py b/data/leaderboard_statistics.py index 377cbca..b7124d4 100644 --- a/data/leaderboard_statistics.py +++ b/data/leaderboard_statistics.py @@ -22,7 +22,19 @@ not a claim that all members are mutually indistinguishable, and not a property of any entry on its own. -Both require per-instance outcomes. Entries without them get neither number, and +Both require per-instance outcomes, and both are computed from those outcomes +rather than from the published rate. That makes the two artifacts checkable +against each other, so they are checked: an entry is only annotated when the +resolve rate implied by its per-instance results agrees with its published rate +to within one instance (100/n percentage points). A larger disagreement means +the two disagree about the outcome of at least one instance, and there is no way +to tell from here which of them is right, so the entry gets no numbers and the +reason is recorded rather than the disagreement being averaged away. This is not +hypothetical: on the current board two submissions publish per-instance files +whose implied rates are 0.00 % and 10.20 % against published rates of 69.60 % +and 52.80 %. + +Entries without per-instance results get neither number, and the caller is expected to render that as "not published" rather than as a blank that reads like a pass. Where the highest entry on a board publishes nothing, the anchor is the highest entry that does, and the summary says who it is and how @@ -59,14 +71,20 @@ def _binom_cdf_half(k: int, n: int) -> float: return sum(math.comb(n, i) for i in range(k + 1)) / (2.0**n) -def mcnemar_exact(a: dict, b: dict) -> float: +def mcnemar_exact(a: dict, b: dict) -> float | None: """Two-sided exact McNemar p-value over the instances both entries report. a and b map instance id -> bool (resolved). Only shared instances are used, because the test is paired: an instance one entry never attempted carries no - information about the difference between them. + information about the difference between them. Returns None when there is no + shared instance at all, which is "not comparable", not "not different". """ shared = a.keys() & b.keys() + if not shared: + # No paired evidence at all. Returning 1.0 here would put two entries + # that were never scored on a common instance into the same group, i.e. + # report absence of data as absence of difference. + return None a_only = sum(1 for i in shared if a[i] and not b[i]) b_only = sum(1 for i in shared if b[i] and not a[i]) n = a_only + b_only @@ -76,9 +94,13 @@ def mcnemar_exact(a: dict, b: dict) -> float: def binomial_se_pp(resolved: int, n: int) -> float: - """Standard error of a resolve rate, in percentage points.""" + """Standard error of a resolve rate, in percentage points. + + An empty sample raises rather than returning 0.0: no observations is not the + same fact as no error, and a zero in this column would read as the latter. + """ if n <= 0: - return 0.0 + raise ValueError("standard error of an empty sample is undefined, not zero") p = resolved / n return 100.0 * math.sqrt(p * (1.0 - p) / n) @@ -90,20 +112,62 @@ def _outcomes(entry: dict) -> dict | None: return {k: bool(v.get("resolved")) for k, v in details.items()} +def usable_outcomes(entry: dict) -> tuple[dict | None, str | None]: + """Per-instance outcomes for an entry, or None with the reason they are unusable. + + Reasons are named rather than collapsed into a blank, because "nothing was + published" and "what was published contradicts the headline number" are + different facts about a submission and only one of them is the submitter's + silence. + """ + outcomes = _outcomes(entry) + if outcomes is None: + return None, "not_published" + + n = len(outcomes) + implied = 100.0 * sum(outcomes.values()) / n + published = entry.get("resolved") + if published is None: + return None, "no_published_rate" + + # One instance is the finest step the published rate can move in, so a + # disagreement smaller than that is rounding or a denominator that differs + # by one, and anything larger means the two artifacts disagree about the + # outcome of at least one instance. + if abs(implied - float(published)) >= 100.0 / n: + entry["stats_implied_resolved"] = round(implied, 2) + return None, "inconsistent_with_published_rate" + + return outcomes, None + + def annotate(leaderboard: dict, alpha: float = ALPHA) -> dict: """Add resolved_se / n_instances / tie_group in place. Returns a summary.""" results = leaderboard.get("results", []) ranked = sorted(results, key=lambda r: -(float(r.get("resolved") or 0))) + usable: dict[int, dict] = {} + excluded: list[tuple[str, str]] = [] for entry in results: - outcomes = _outcomes(entry) + outcomes, reason = usable_outcomes(entry) if outcomes is None: + if reason != "not_published": + entry["stats_excluded"] = reason + excluded.append((entry.get("model_display") or entry.get("name"), reason)) continue + usable[id(entry)] = outcomes n = len(outcomes) + resolved = sum(outcomes.values()) entry["n_instances"] = n - entry["resolved_se"] = round(binomial_se_pp(sum(outcomes.values()), n), 2) - - measurable = [r for r in ranked if _outcomes(r) is not None] + if resolved in (0, n): + # sqrt(p(1-p)/n) is exactly zero at a boundary rate. That is a + # property of the estimator, not a measurement of this entry, and + # printing 0.00 in an error column would claim the opposite. + entry["stats_excluded"] = "degenerate_rate" + else: + entry["resolved_se"] = round(binomial_se_pp(resolved, n), 2) + + measurable = [r for r in ranked if id(r) in usable] summary = { "name": leaderboard.get("name"), "entries": len(results), @@ -113,6 +177,8 @@ def annotate(leaderboard: dict, alpha: float = ALPHA) -> dict: "top_tie_group_size": 0, "anchor": None, "unmeasured_above_anchor": 0, + "excluded": excluded, + "not_comparable": 0, } if not measurable: return summary @@ -130,11 +196,18 @@ def annotate(leaderboard: dict, alpha: float = ALPHA) -> dict: while pending: group += 1 anchor, rest = pending[0], pending[1:] - anchor_outcomes = _outcomes(anchor) + anchor_outcomes = usable[id(anchor)] anchor["tie_group"] = group pending = [] for entry in rest: - if mcnemar_exact(anchor_outcomes, _outcomes(entry)) >= alpha: + p = mcnemar_exact(anchor_outcomes, usable[id(entry)]) + if p is None: + # Shares no instance with this anchor: it cannot join the group + # and it cannot be said to differ from it either. It stays in + # the queue and will anchor a group of its own. + summary["not_comparable"] += 1 + pending.append(entry) + elif p >= alpha: entry["tie_group"] = group else: pending.append(entry) @@ -250,6 +323,72 @@ def selftest() -> int: if mcnemar_exact(short, long_) != 1.0: failures.append("instances missing from one entry were treated as failures") + # An empty sample has no standard error, and must not report one as zero. + try: + binomial_se_pp(0, 0) + failures.append("an empty sample was given a standard error") + except ValueError: + pass + + # Per-instance results that contradict the published rate are not averaged + # away: the entry gets no numbers, and the reason is recorded by name. + liar = _entry("liar", [1] * 51 + [0] * 449) # implies 10.2 % + liar["resolved"] = 52.8 # but the board says 52.8 % + lb5 = {"name": "t5", "results": [_entry("lead", lead), liar, _entry("lag", lag)]} + s5 = annotate(lb5) + if "resolved_se" in liar or "tie_group" in liar: + failures.append("an entry contradicting its published rate was given statistics") + if liar.get("stats_excluded") != "inconsistent_with_published_rate": + failures.append(f"exclusion reason was {liar.get('stats_excluded')!r}") + if liar.get("stats_implied_resolved") != 10.2: + failures.append(f"implied rate was {liar.get('stats_implied_resolved')!r}, expected 10.2") + if ("liar", "inconsistent_with_published_rate") not in s5["excluded"]: + failures.append("the exclusion was not reported in the summary") + + # A disagreement smaller than one instance is a rounding or denominator + # difference, not a contradiction, and must not drop the entry. + rounded = _entry("rounded", [1] * 324 + [0] * 176) # implies 64.80 % + rounded["resolved"] = 64.93 # published over 499 + lb6 = {"name": "t6", "results": [rounded, _entry("lag", lag)]} + annotate(lb6) + if rounded.get("resolved_se") is None or rounded.get("tie_group") != 1: + failures.append("a sub-instance rounding difference dropped the entry") + + # ... and the threshold is exactly one instance, so a disagreement just over + # it is excluded. Without this the gate could be loosened by an order of + # magnitude and every test above would still pass. + over = _entry("over", [1] * 324 + [0] * 176) # implies 64.80 % + over["resolved"] = 65.05 # 0.25 pp away, one instance is 0.20 + lb6b = {"name": "t6b", "results": [over, _entry("lag", lag)]} + annotate(lb6b) + if over.get("stats_excluded") != "inconsistent_with_published_rate": + failures.append("a disagreement larger than one instance was accepted") + + # At a boundary rate the binomial SE is exactly zero, which is a property of + # the estimator. It is withheld, but the entry is still compared. + perfect = _entry("perfect", [1] * 500) + lb7 = {"name": "t7", "results": [perfect, _entry("lag", lag)]} + annotate(lb7) + if "resolved_se" in perfect: + failures.append("a boundary rate published a zero standard error") + if perfect.get("stats_excluded") != "degenerate_rate" or perfect.get("tie_group") != 1: + failures.append("a boundary-rate entry was dropped from the comparison as well") + + # Two entries scored on disjoint instance sets are not comparable, and + # absence of paired evidence must not be rendered as a tie. + left = {"name": "left", "resolved": 60.0, + "per_instance_details": {f"a{i}": {"resolved": i < 60} for i in range(100)}} + right = {"name": "right", "resolved": 55.0, + "per_instance_details": {f"b{i}": {"resolved": i < 55} for i in range(100)}} + lb8 = {"name": "t8", "results": [left, right]} + s8 = annotate(lb8) + if left.get("tie_group") == right.get("tie_group"): + failures.append("entries with no shared instances were put in one group") + if s8["not_comparable"] != 1: + failures.append(f"not_comparable was {s8['not_comparable']}, expected 1") + if mcnemar_exact({"a": True}, {"b": True}) is not None: + failures.append("a comparison with no shared instance returned a p-value") + if failures: print("SELFTEST FAILED") for f in failures: @@ -258,7 +397,10 @@ def selftest() -> int: print("selftest passed: identical entries never separated, planted flips detected, " "balanced discordance not called a difference, SE matches the binomial formula, " "the anchor is the highest measurable entry and is named, missing per-instance " - "results stay empty") + "results stay empty, per-instance results contradicting the published rate are " + "excluded by name, a sub-instance rounding difference is not a contradiction, a " + "boundary rate withholds the zero SE, an empty sample raises, and disjoint " + "instance sets are not a tie") return 0 @@ -276,6 +418,8 @@ def main() -> int: for s in annotate_all(boards): print(f"{s['name']:<14} entries={s['entries']:>4} with per-instance={s['with_per_instance']:>4} " f"grouped={s['grouped']:>4} groups={s['groups']:>3} top group={s['top_tie_group_size']:>3}") + for name, reason in s["excluded"]: + print(f"{'':<14} excluded: {name}: {reason}") return 0 diff --git a/js/mainResults.js b/js/mainResults.js index 8a2659d..acb09af 100644 --- a/js/mainResults.js +++ b/js/mainResults.js @@ -10,6 +10,27 @@ const sortState = { field: 'resolved', direction: 'desc' }; // as "the smallest error" or "the top group". const MISSING = Symbol('missing'); +// Why a cell is empty is itself a fact about the submission, so the dash says +// which of the reasons applies instead of collapsing them into silence. +const STAT_GAP = { + inconsistent_with_published_rate: + 'The per-instance results published for this entry imply a different resolve rate ' + + 'than the one on the board, so neither number is computed from them', + no_published_rate: + 'This entry has no published resolve rate to check the per-instance results against', + degenerate_rate: + 'Every instance has the same outcome, where sqrt(p(1-p)/n) is exactly zero. That is a ' + + 'property of the formula, not a measurement of this entry, so no error is shown', +}; + +function statGapTitle(item, fallback) { + const reason = STAT_GAP[item.stats_excluded]; + if (!reason) return fallback; + return item.stats_implied_resolved != null + ? `${reason} (${item.stats_implied_resolved}% implied, ${item.resolved}% published)` + : reason; +} + function escapeAttr(value) { return String(value == null ? '' : value).replace(/[&<>"']/g, ch => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' @@ -304,10 +325,10 @@ function renderLeaderboardTable(leaderboard) { ${withStats ? `${item.resolved_se != null ? `±${parseFloat(item.resolved_se).toFixed(2)}` - : '—'}` : ''} + : `—`}` : ''} ${withStats ? `${item.tie_group != null ? `${item.tie_group}` - : '—'}` : ''} + : `—`}` : ''} ${withHarness ? `${item.instance_cost !== null && item.instance_cost !== undefined && item.instance_cost !== 0 && !isNaN(item.instance_cost) ? '$' + parseFloat(item.instance_cost).toFixed(2) : ''}` : ''} ${withHarness ? ` ${item.trajs_docent && item.trajs_docent !== false ? `` : '-'} From a579699824f83d2493d0545d614ebb433fcf091f Mon Sep 17 00:00:00 2001 From: ipezygj Date: Thu, 20 Aug 2026 13:09:22 +0300 Subject: [PATCH 4/4] Name the inference target the SE column relies on The resolve rate on a board is an exact descriptive fact: every entry is scored on the same fixed set of instances, so 396 of 500 is 79.20 % with nothing left to estimate. sqrt(p(1-p)/n) is a standard error only under a declared target of inference, and the column never declared one. The note and the column tooltip now separate four objects that were running together: the observed proportion on a fixed set (exact, descriptive), the binomial SE (model-based, generalising to comparable tasks under an exchangeability assumption), the paired McNemar comparison on shared instances, and repeatability, which a single run cannot observe. The exchangeability assumption is stated rather than implied, and stated against these item sets in particular: they are curated and human-filtered rather than randomly drawn, which makes the assumption questionable, not merely unstated. Wording only. No change to leaderboard_statistics.py, and the computed output is unchanged: Verified still reports 38 of 180 entries with per-instance results and 6 in group 1 anchored on Claude 4.5 Opus. --- js/mainResults.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/js/mainResults.js b/js/mainResults.js index acb09af..a36913b 100644 --- a/js/mainResults.js +++ b/js/mainResults.js @@ -281,7 +281,7 @@ function renderLeaderboardTable(leaderboard) { Model Agent % Resolved - ${withStats ? '± SE' : ''} + ${withStats ? '± SE' : ''} ${withStats ? 'Tie' : ''} ${withHarness ? 'Avg. $' : ''} ${withHarness ? 'Trajs' : ''} @@ -358,9 +358,19 @@ function renderLeaderboardTable(leaderboard) { ${withStats ? `

- ± SE is the standard error of a single entry's resolve rate, - sqrt(p(1-p)/n) in percentage points, over the - instances that entry was scored on. Two entries are scored on the same + ± SE is sqrt(p(1-p)/n) in + percentage points over the instances that entry was scored on, and it is not the + uncertainty of the number beside it. Every entry on a board is scored on the same + fixed set of instances, and on that fixed set the resolve rate is an exact + descriptive fact: 396 of 500 is 79.20 %, with nothing left to estimate. That + quantity becomes a standard error only once the target of the inference is named, + and the target here is a wider population of comparable tasks of which this item + set is treated as an exchangeable sample. That assumption is the column's content, + and it is worth stating plainly that these item sets are curated and + human-filtered rather than randomly drawn, which makes the assumption questionable + rather than merely unstated. Read the column as how far the rate would move across + comparable item sets under that model, not as error bars on the benchmark result. + Two entries are scored on the same instances, so whether they differ is a paired question and overlapping SEs are not the answer to it. Tie answers that paired question: entries an exact two-sided McNemar test