diff --git a/build.py b/build.py
index 6127e4d..de66e83 100644
--- a/build.py
+++ b/build.py
@@ -2,6 +2,7 @@
import json
import pathlib
import shutil
+import sys
from flask import Flask
from flask_flatpages import FlatPages
from jinja2 import Environment, FileSystemLoader, select_autoescape
@@ -11,6 +12,9 @@
TEMPLATES = ROOT / "templates"
DIST = ROOT / "dist"
+sys.path.insert(0, str(ROOT / "data"))
+from leaderboard_statistics import annotate_all # noqa: E402
+
def get_pages():
pages = {}
pages_dir = TEMPLATES / "pages"
@@ -126,6 +130,31 @@ def main() -> None:
# load data
with open(ROOT / "data/leaderboards.json", "r") as f:
leaderboards = json.load(f)
+
+ # Sampling error and significance groups, computed here rather than stored,
+ # so that leaderboards.json stays the record of what was submitted and these
+ # stay a function of it. Boards without per-instance results are untouched.
+ for summary in annotate_all(
+ leaderboards["leaderboards"] if isinstance(leaderboards, dict) else leaderboards
+ ):
+ line = (f"stats: {summary['name']}: {summary['with_per_instance']}/{summary['entries']} "
+ f"entries with per-instance results")
+ if summary["groups"]:
+ line += (f", {summary['groups']} significance groups, "
+ f"{summary['top_tie_group_size']} in group 1 anchored on "
+ f"{summary['anchor']}")
+ if summary["unmeasured_above_anchor"]:
+ line += f" ({summary['unmeasured_above_anchor']} higher entries publish none)"
+ if summary["not_comparable"]:
+ line += (f", {summary['not_comparable']} entries share no instance with the "
+ "anchor of their group")
+ print(line)
+ # An entry dropped for contradicting its own published rate is a defect
+ # in the submitted data, not a quiet gap in a column, so the build says
+ # so by name rather than leaving a dash on the page as the only trace.
+ for name, reason in summary["excluded"]:
+ print(f"stats: {summary['name']}: {name}: {reason}")
+
with open(ROOT / "data/press.json", "r") as f:
press = json.load(f)
press = sorted(press, key=lambda x: x["date"], reverse=True)
diff --git a/css/leaderboard.css b/css/leaderboard.css
index 90edebb..e8bddfe 100644
--- a/css/leaderboard.css
+++ b/css/leaderboard.css
@@ -782,10 +782,49 @@
min-width: 66rem;
}
+/* The two statistics columns add sized width, so the point at which the wrapper
+ starts scrolling instead of squeezing Model moves with them. */
+#leaderboard-container .data-table[data-stats="true"] {
+ min-width: 74rem;
+}
+
#leaderboard-container col.cw-select { width: 2rem; }
#leaderboard-container col.cw-rank { width: 3.1rem; }
#leaderboard-container col.cw-agent { width: 12rem; }
#leaderboard-container col.cw-resolved { width: 7.25rem; }
+/* Statistics columns: the error on one entry, and the paired group it sits in */
+#leaderboard-container .stat-cell .number,
+#leaderboard-container .stat-cell .text-muted {
+ font-variant-numeric: tabular-nums;
+}
+
+#leaderboard-container .tie-group {
+ display: inline-block;
+ min-width: 1.5rem;
+ padding: 0.05rem 0.35rem;
+ border-radius: var(--radius-sm, 0.25rem);
+ font-variant-numeric: tabular-nums;
+ color: var(--color-text-muted);
+}
+
+/* Group 1 is the one the headline number is read off, so it is the one worth
+ seeing at a glance. Emphasis, not a claim of quality. The tint is the same
+ accent the site already defines for both themes, so it stays visible in dark
+ mode instead of disappearing into the background. */
+#leaderboard-container .tie-group-top {
+ background-color: var(--color-accent-light);
+ color: var(--color-text);
+ font-weight: 600;
+}
+
+.stats-note {
+ margin: 0.5rem 0 0;
+ font-size: 0.8125rem;
+ line-height: 1.5;
+}
+
+#leaderboard-container col.cw-se { width: 4.75rem; }
+#leaderboard-container col.cw-tie { width: 3.5rem; }
#leaderboard-container col.cw-cost { width: 5.25rem; }
#leaderboard-container col.cw-trajs { width: 4.5rem; }
#leaderboard-container col.cw-org { width: 4.75rem; }
diff --git a/data/leaderboard_statistics.py b/data/leaderboard_statistics.py
new file mode 100644
index 0000000..b7124d4
--- /dev/null
+++ b/data/leaderboard_statistics.py
@@ -0,0 +1,427 @@
+#!/usr/bin/env python3
+"""Sampling error and significance groups for leaderboard entries.
+
+Two numbers are added to every entry that publishes per-instance results:
+
+ resolved_se the binomial standard error of THAT entry's resolve rate over
+ the instances it was scored on, in percentage points:
+ 100 * sqrt(p*(1-p)/n). It is the sampling error of one rate on
+ a fixed instance set. It is NOT the error of a difference
+ between two entries: two entries are scored on the same
+ instances, so their difference is a paired quantity with a
+ smaller error than these two marginals suggest. Read this
+ column as "how precisely is this one number measured", never
+ as "these two entries overlap, therefore they are tied".
+
+ tie_group the significance group, from the paired comparison that the SE
+ column deliberately does not perform. Entries are ordered by
+ resolve rate; the highest ungrouped entry anchors a group, and
+ every entry an exact two-sided McNemar test cannot separate
+ from that anchor at alpha = 0.05 joins it. Group membership is
+ therefore a statement about the comparison with the anchor,
+ not a claim that all members are mutually indistinguishable,
+ and not a property of any entry on its own.
+
+Both require per-instance outcomes, and both are computed from those outcomes
+rather than from the published rate. That makes the two artifacts checkable
+against each other, so they are checked: an entry is only annotated when the
+resolve rate implied by its per-instance results agrees with its published rate
+to within one instance (100/n percentage points). A larger disagreement means
+the two disagree about the outcome of at least one instance, and there is no way
+to tell from here which of them is right, so the entry gets no numbers and the
+reason is recorded rather than the disagreement being averaged away. This is not
+hypothetical: on the current board two submissions publish per-instance files
+whose implied rates are 0.00 % and 10.20 % against published rates of 69.60 %
+and 52.80 %.
+
+Entries without per-instance results get neither number, and
+the caller is expected to render that as "not published" rather than as a blank
+that reads like a pass. Where the highest entry on a board publishes nothing, the
+anchor is the highest entry that does, and the summary says who it is and how
+many unmeasured entries rank above it — a group is a statement about a comparison,
+and it cannot be read without knowing what the comparison was against.
+
+No multiplicity correction is applied to the comparisons against an anchor. A
+correction would make separation harder and the groups larger, so the group
+sizes produced here are a lower bound on how much of a board is statistically
+indistinguishable.
+
+Neither number is a repeatability estimate. Both condition on the single run each
+entry reports. A rerun of the same system on the same instances is not observable
+in this data, so nothing here bounds how much a score would move on a second run,
+and neither column may be read as the uncertainty of the ranking itself.
+
+ python leaderboard_statistics.py --selftest
+ python leaderboard_statistics.py leaderboards.json # report, writes nothing
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import pathlib
+import sys
+
+ALPHA = 0.05
+
+
+def _binom_cdf_half(k: int, n: int) -> float:
+ """P(X <= k) for X ~ Binomial(n, 0.5), computed exactly."""
+ return sum(math.comb(n, i) for i in range(k + 1)) / (2.0**n)
+
+
+def mcnemar_exact(a: dict, b: dict) -> float | None:
+ """Two-sided exact McNemar p-value over the instances both entries report.
+
+ a and b map instance id -> bool (resolved). Only shared instances are used,
+ because the test is paired: an instance one entry never attempted carries no
+ information about the difference between them. Returns None when there is no
+ shared instance at all, which is "not comparable", not "not different".
+ """
+ shared = a.keys() & b.keys()
+ if not shared:
+ # No paired evidence at all. Returning 1.0 here would put two entries
+ # that were never scored on a common instance into the same group, i.e.
+ # report absence of data as absence of difference.
+ return None
+ a_only = sum(1 for i in shared if a[i] and not b[i])
+ b_only = sum(1 for i in shared if b[i] and not a[i])
+ n = a_only + b_only
+ if n == 0:
+ return 1.0
+ return min(1.0, 2.0 * _binom_cdf_half(min(a_only, b_only), n))
+
+
+def binomial_se_pp(resolved: int, n: int) -> float:
+ """Standard error of a resolve rate, in percentage points.
+
+ An empty sample raises rather than returning 0.0: no observations is not the
+ same fact as no error, and a zero in this column would read as the latter.
+ """
+ if n <= 0:
+ raise ValueError("standard error of an empty sample is undefined, not zero")
+ p = resolved / n
+ return 100.0 * math.sqrt(p * (1.0 - p) / n)
+
+
+def _outcomes(entry: dict) -> dict | None:
+ details = entry.get("per_instance_details")
+ if not details:
+ return None
+ return {k: bool(v.get("resolved")) for k, v in details.items()}
+
+
+def usable_outcomes(entry: dict) -> tuple[dict | None, str | None]:
+ """Per-instance outcomes for an entry, or None with the reason they are unusable.
+
+ Reasons are named rather than collapsed into a blank, because "nothing was
+ published" and "what was published contradicts the headline number" are
+ different facts about a submission and only one of them is the submitter's
+ silence.
+ """
+ outcomes = _outcomes(entry)
+ if outcomes is None:
+ return None, "not_published"
+
+ n = len(outcomes)
+ implied = 100.0 * sum(outcomes.values()) / n
+ published = entry.get("resolved")
+ if published is None:
+ return None, "no_published_rate"
+
+ # One instance is the finest step the published rate can move in, so a
+ # disagreement smaller than that is rounding or a denominator that differs
+ # by one, and anything larger means the two artifacts disagree about the
+ # outcome of at least one instance.
+ if abs(implied - float(published)) >= 100.0 / n:
+ entry["stats_implied_resolved"] = round(implied, 2)
+ return None, "inconsistent_with_published_rate"
+
+ return outcomes, None
+
+
+def annotate(leaderboard: dict, alpha: float = ALPHA) -> dict:
+ """Add resolved_se / n_instances / tie_group in place. Returns a summary."""
+ results = leaderboard.get("results", [])
+ ranked = sorted(results, key=lambda r: -(float(r.get("resolved") or 0)))
+
+ usable: dict[int, dict] = {}
+ excluded: list[tuple[str, str]] = []
+ for entry in results:
+ outcomes, reason = usable_outcomes(entry)
+ if outcomes is None:
+ if reason != "not_published":
+ entry["stats_excluded"] = reason
+ excluded.append((entry.get("model_display") or entry.get("name"), reason))
+ continue
+ usable[id(entry)] = outcomes
+ n = len(outcomes)
+ resolved = sum(outcomes.values())
+ entry["n_instances"] = n
+ if resolved in (0, n):
+ # sqrt(p(1-p)/n) is exactly zero at a boundary rate. That is a
+ # property of the estimator, not a measurement of this entry, and
+ # printing 0.00 in an error column would claim the opposite.
+ entry["stats_excluded"] = "degenerate_rate"
+ else:
+ entry["resolved_se"] = round(binomial_se_pp(resolved, n), 2)
+
+ measurable = [r for r in ranked if id(r) in usable]
+ summary = {
+ "name": leaderboard.get("name"),
+ "entries": len(results),
+ "with_per_instance": len(measurable),
+ "grouped": 0,
+ "groups": 0,
+ "top_tie_group_size": 0,
+ "anchor": None,
+ "unmeasured_above_anchor": 0,
+ "excluded": excluded,
+ "not_comparable": 0,
+ }
+ if not measurable:
+ return summary
+
+ # Group 1 is anchored on the highest entry that publishes per-instance
+ # results, which is not always the highest entry on the board. Whoever it is
+ # gets named, because a group is a statement about a comparison and the
+ # reader cannot check it without knowing what the comparison was against.
+ anchor_index = ranked.index(measurable[0])
+ summary["anchor"] = measurable[0].get("model_display") or measurable[0].get("name")
+ summary["unmeasured_above_anchor"] = anchor_index
+
+ pending = measurable
+ group = 0
+ while pending:
+ group += 1
+ anchor, rest = pending[0], pending[1:]
+ anchor_outcomes = usable[id(anchor)]
+ anchor["tie_group"] = group
+ pending = []
+ for entry in rest:
+ p = mcnemar_exact(anchor_outcomes, usable[id(entry)])
+ if p is None:
+ # Shares no instance with this anchor: it cannot join the group
+ # and it cannot be said to differ from it either. It stays in
+ # the queue and will anchor a group of its own.
+ summary["not_comparable"] += 1
+ pending.append(entry)
+ elif p >= alpha:
+ entry["tie_group"] = group
+ else:
+ pending.append(entry)
+ if group == 1:
+ summary["top_tie_group_size"] = 1 + sum(
+ 1 for e in rest if e.get("tie_group") == 1
+ )
+
+ summary["groups"] = group
+ summary["grouped"] = sum(1 for r in results if r.get("tie_group"))
+ leaderboard["statistics"] = dict(summary, alpha=alpha)
+ return summary
+
+
+def annotate_all(leaderboards: list[dict], alpha: float = ALPHA) -> list[dict]:
+ return [annotate(lb, alpha) for lb in leaderboards]
+
+
+# ------------------------------------------------------------------ selftest
+
+def _entry(name: str, outcomes: list[int]) -> dict:
+ return {
+ "name": name,
+ "resolved": 100.0 * sum(outcomes) / len(outcomes),
+ "per_instance_details": {
+ f"i{i}": {"resolved": bool(v)} for i, v in enumerate(outcomes)
+ },
+ }
+
+
+def selftest() -> int:
+ failures = []
+
+ # An entry compared with itself is never separated, at any size.
+ same = [i % 3 == 0 for i in range(500)]
+ a = {f"i{i}": v for i, v in enumerate(same)}
+ if mcnemar_exact(a, dict(a)) != 1.0:
+ failures.append("identical entries were separated")
+
+ # The control above must not pass for lack of power: plant flips one way.
+ b = dict(a)
+ flipped = 0
+ for k in list(b):
+ if b[k] and flipped < 40:
+ b[k] = False
+ flipped += 1
+ if mcnemar_exact(a, b) >= 1e-6:
+ failures.append("40 planted one-directional flips were not detected")
+
+ # Discordance that is balanced is not evidence of a difference, however big.
+ c, d = {}, {}
+ for i in range(200):
+ c[f"i{i}"], d[f"i{i}"] = (i % 2 == 0), (i % 2 == 1)
+ if mcnemar_exact(c, d) < ALPHA:
+ failures.append("200 balanced discordant pairs were called a difference")
+
+ # SE is the textbook binomial one, and zero variance means zero error.
+ if abs(binomial_se_pp(250, 500) - 100 * math.sqrt(0.25 / 500)) > 1e-9:
+ failures.append("binomial SE does not match sqrt(p(1-p)/n)")
+ if binomial_se_pp(500, 500) != 0.0:
+ failures.append("a perfect score did not get a zero SE")
+
+ # Grouping: two indistinguishable leaders and one clearly worse entry give
+ # group 1 = {leader, near-tie} and group 2 = {laggard}.
+ lead = [1] * 260 + [0] * 240
+ near = lead[:]
+ near[0], near[1], near[259] = 0, 0, 0
+ near[300], near[301] = 1, 1
+ lag = [1] * 150 + [0] * 350
+ lb = {"name": "t", "results": [_entry("lead", lead), _entry("near", near), _entry("lag", lag)]}
+ s = annotate(lb)
+ got = {r["name"]: r.get("tie_group") for r in lb["results"]}
+ if got != {"lead": 1, "near": 1, "lag": 2}:
+ failures.append(f"grouping was {got}, expected lead/near in 1 and lag in 2")
+ if s["top_tie_group_size"] != 2:
+ failures.append(f"top group size was {s['top_tie_group_size']}, expected 2")
+
+ # An entry with no per-instance results gets no numbers and no group, and it
+ # does not shift anyone else's group.
+ lb2 = {"name": "t2", "results": [_entry("lead", lead), {"name": "silent", "resolved": 40.0},
+ _entry("lag", lag)]}
+ annotate(lb2)
+ silent = lb2["results"][1]
+ if "tie_group" in silent or "resolved_se" in silent:
+ failures.append("an entry without per-instance results was given statistics")
+ if {r["name"]: r.get("tie_group") for r in lb2["results"] if "per_instance_details" in r} != {
+ "lead": 1, "lag": 2}:
+ failures.append("a silent entry changed the groups of the others")
+
+ # A board whose top entry publishes nothing still gets groups, anchored on
+ # the highest entry that does — and that entry is named, with a count of how
+ # many unmeasured entries rank above it, so the anchor is never implicit.
+ lb3 = {"name": "t3", "results": [{"name": "top", "resolved": 99.0}, _entry("lead", lead),
+ _entry("lag", lag)]}
+ s3 = annotate(lb3)
+ if s3["anchor"] != "lead" or s3["unmeasured_above_anchor"] != 1:
+ failures.append(f"anchor was {s3['anchor']!r} with {s3['unmeasured_above_anchor']} "
+ "unmeasured above it, expected 'lead' with 1")
+ if lb3["results"][0].get("tie_group") is not None:
+ failures.append("an entry with no per-instance results was put in a group")
+ if s3["groups"] != 2:
+ failures.append(f"expected 2 groups under an unmeasured top entry, got {s3['groups']}")
+
+ # A board where nothing is measurable produces no groups and no anchor.
+ lb4 = {"name": "t4", "results": [{"name": "a", "resolved": 9.0}, {"name": "b", "resolved": 8.0}]}
+ s4 = annotate(lb4)
+ if s4["groups"] or s4["anchor"] is not None or "statistics" in lb4:
+ failures.append("a board with no per-instance results was given statistics")
+
+ # Only shared instances are compared.
+ short = {"i0": True, "i1": False}
+ long_ = {"i0": True, "i1": False, "i2": True, "i3": True}
+ if mcnemar_exact(short, long_) != 1.0:
+ failures.append("instances missing from one entry were treated as failures")
+
+ # An empty sample has no standard error, and must not report one as zero.
+ try:
+ binomial_se_pp(0, 0)
+ failures.append("an empty sample was given a standard error")
+ except ValueError:
+ pass
+
+ # Per-instance results that contradict the published rate are not averaged
+ # away: the entry gets no numbers, and the reason is recorded by name.
+ liar = _entry("liar", [1] * 51 + [0] * 449) # implies 10.2 %
+ liar["resolved"] = 52.8 # but the board says 52.8 %
+ lb5 = {"name": "t5", "results": [_entry("lead", lead), liar, _entry("lag", lag)]}
+ s5 = annotate(lb5)
+ if "resolved_se" in liar or "tie_group" in liar:
+ failures.append("an entry contradicting its published rate was given statistics")
+ if liar.get("stats_excluded") != "inconsistent_with_published_rate":
+ failures.append(f"exclusion reason was {liar.get('stats_excluded')!r}")
+ if liar.get("stats_implied_resolved") != 10.2:
+ failures.append(f"implied rate was {liar.get('stats_implied_resolved')!r}, expected 10.2")
+ if ("liar", "inconsistent_with_published_rate") not in s5["excluded"]:
+ failures.append("the exclusion was not reported in the summary")
+
+ # A disagreement smaller than one instance is a rounding or denominator
+ # difference, not a contradiction, and must not drop the entry.
+ rounded = _entry("rounded", [1] * 324 + [0] * 176) # implies 64.80 %
+ rounded["resolved"] = 64.93 # published over 499
+ lb6 = {"name": "t6", "results": [rounded, _entry("lag", lag)]}
+ annotate(lb6)
+ if rounded.get("resolved_se") is None or rounded.get("tie_group") != 1:
+ failures.append("a sub-instance rounding difference dropped the entry")
+
+ # ... and the threshold is exactly one instance, so a disagreement just over
+ # it is excluded. Without this the gate could be loosened by an order of
+ # magnitude and every test above would still pass.
+ over = _entry("over", [1] * 324 + [0] * 176) # implies 64.80 %
+ over["resolved"] = 65.05 # 0.25 pp away, one instance is 0.20
+ lb6b = {"name": "t6b", "results": [over, _entry("lag", lag)]}
+ annotate(lb6b)
+ if over.get("stats_excluded") != "inconsistent_with_published_rate":
+ failures.append("a disagreement larger than one instance was accepted")
+
+ # At a boundary rate the binomial SE is exactly zero, which is a property of
+ # the estimator. It is withheld, but the entry is still compared.
+ perfect = _entry("perfect", [1] * 500)
+ lb7 = {"name": "t7", "results": [perfect, _entry("lag", lag)]}
+ annotate(lb7)
+ if "resolved_se" in perfect:
+ failures.append("a boundary rate published a zero standard error")
+ if perfect.get("stats_excluded") != "degenerate_rate" or perfect.get("tie_group") != 1:
+ failures.append("a boundary-rate entry was dropped from the comparison as well")
+
+ # Two entries scored on disjoint instance sets are not comparable, and
+ # absence of paired evidence must not be rendered as a tie.
+ left = {"name": "left", "resolved": 60.0,
+ "per_instance_details": {f"a{i}": {"resolved": i < 60} for i in range(100)}}
+ right = {"name": "right", "resolved": 55.0,
+ "per_instance_details": {f"b{i}": {"resolved": i < 55} for i in range(100)}}
+ lb8 = {"name": "t8", "results": [left, right]}
+ s8 = annotate(lb8)
+ if left.get("tie_group") == right.get("tie_group"):
+ failures.append("entries with no shared instances were put in one group")
+ if s8["not_comparable"] != 1:
+ failures.append(f"not_comparable was {s8['not_comparable']}, expected 1")
+ if mcnemar_exact({"a": True}, {"b": True}) is not None:
+ failures.append("a comparison with no shared instance returned a p-value")
+
+ if failures:
+ print("SELFTEST FAILED")
+ for f in failures:
+ print(" -", f)
+ return 1
+ print("selftest passed: identical entries never separated, planted flips detected, "
+ "balanced discordance not called a difference, SE matches the binomial formula, "
+ "the anchor is the highest measurable entry and is named, missing per-instance "
+ "results stay empty, per-instance results contradicting the published rate are "
+ "excluded by name, a sub-instance rounding difference is not a contradiction, a "
+ "boundary rate withholds the zero SE, an empty sample raises, and disjoint "
+ "instance sets are not a tie")
+ return 0
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("json", nargs="?", help="leaderboards.json to report on (not modified)")
+ ap.add_argument("--selftest", action="store_true")
+ args = ap.parse_args()
+
+ if args.selftest or not args.json:
+ return selftest()
+
+ data = json.loads(pathlib.Path(args.json).read_text(encoding="utf-8"))
+ boards = data["leaderboards"] if isinstance(data, dict) else data
+ for s in annotate_all(boards):
+ print(f"{s['name']:<14} entries={s['entries']:>4} with per-instance={s['with_per_instance']:>4} "
+ f"grouped={s['grouped']:>4} groups={s['groups']:>3} top group={s['top_tie_group_size']:>3}")
+ for name, reason in s["excluded"]:
+ print(f"{'':<14} excluded: {name}: {reason}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/js/mainResults.js b/js/mainResults.js
index a703a78..a36913b 100644
--- a/js/mainResults.js
+++ b/js/mainResults.js
@@ -5,6 +5,32 @@ let leaderboardData = null;
const sortState = { field: 'resolved', direction: 'desc' };
+// A cell that has nothing in it is not a small value: it sorts last whichever
+// way the arrow points, so that "no per-instance results published" never reads
+// as "the smallest error" or "the top group".
+const MISSING = Symbol('missing');
+
+// Why a cell is empty is itself a fact about the submission, so the dash says
+// which of the reasons applies instead of collapsing them into silence.
+const STAT_GAP = {
+ inconsistent_with_published_rate:
+ 'The per-instance results published for this entry imply a different resolve rate '
+ + 'than the one on the board, so neither number is computed from them',
+ no_published_rate:
+ 'This entry has no published resolve rate to check the per-instance results against',
+ degenerate_rate:
+ 'Every instance has the same outcome, where sqrt(p(1-p)/n) is exactly zero. That is a '
+ + 'property of the formula, not a measurement of this entry, so no error is shown',
+};
+
+function statGapTitle(item, fallback) {
+ const reason = STAT_GAP[item.stats_excluded];
+ if (!reason) return fallback;
+ return item.stats_implied_resolved != null
+ ? `${reason} (${item.stats_implied_resolved}% implied, ${item.resolved}% published)`
+ : reason;
+}
+
function escapeAttr(value) {
return String(value == null ? '' : value).replace(/[&<>"']/g, ch => ({
'&': '&', '<': '<', '>': '>', '"': '"', "'": '''
@@ -46,6 +72,11 @@ function sortItems(a, b, field, direction) {
return item.trajs_docent && item.trajs_docent !== false ? 1 : 0;
case 'release':
return (item['mini-swe-agent_version'] || '').toLowerCase();
+ // Entries without per-instance results have no SE and no group.
+ case 'resolved_se':
+ return item.resolved_se == null ? MISSING : parseFloat(item.resolved_se);
+ case 'tie_group':
+ return item.tie_group == null ? MISSING : parseInt(item.tie_group, 10);
default:
return '';
}
@@ -53,7 +84,12 @@ function sortItems(a, b, field, direction) {
const av = getValue(a, field);
const bv = getValue(b, field);
-
+
+ if (av === MISSING || bv === MISSING) {
+ if (av === MISSING && bv === MISSING) return 0;
+ return av === MISSING ? 1 : -1;
+ }
+
let result;
if (typeof av === 'number' && typeof bv === 'number') {
result = av - bv;
@@ -204,7 +240,16 @@ function renderLeaderboardTable(leaderboard) {
const displayNames = buildDisplayNames(results);
const modelName = item => item.model_display || displayNames.get(item.name) || item.name;
- const columnCount = 5 + (withSelect ? 1 : 0) + (withHarness ? 3 : 1);
+ // The two statistics columns need per-instance results, which only some
+ // submissions publish. They are shown when the group anchor is the row the
+ // reader is looking at the top of: either nothing on the board outranks it,
+ // or the view has been narrowed to the harness whose runs publish them.
+ // Anywhere else the column would be mostly dashes anchored on a row far
+ // down the page, which says less than leaving it out and explaining why.
+ const stats = leaderboard.statistics;
+ const withStats = !!stats && stats.groups > 0
+ && (withHarness || stats.unmeasured_above_anchor === 0);
+ const columnCount = 5 + (withSelect ? 1 : 0) + (withHarness ? 3 : 1) + (withStats ? 2 : 0);
// The table uses a fixed layout so the narrow columns keep their width no
// matter how many other columns are on screen. Every column is sized here
@@ -215,6 +260,8 @@ function renderLeaderboardTable(leaderboard) {
'
+ ± SE is sqrt(p(1-p)/n) in
+ percentage points over the instances that entry was scored on, and it is not the
+ uncertainty of the number beside it. Every entry on a board is scored on the same
+ fixed set of instances, and on that fixed set the resolve rate is an exact
+ descriptive fact: 396 of 500 is 79.20 %, with nothing left to estimate. That
+ quantity becomes a standard error only once the target of the inference is named,
+ and the target here is a wider population of comparable tasks of which this item
+ set is treated as an exchangeable sample. That assumption is the column's content,
+ and it is worth stating plainly that these item sets are curated and
+ human-filtered rather than randomly drawn, which makes the assumption questionable
+ rather than merely unstated. Read the column as how far the rate would move across
+ comparable item sets under that model, not as error bars on the benchmark result.
+ Two entries are scored on the same
+ instances, so whether they differ is a paired question and overlapping SEs are
+ not the answer to it.
+ Tie answers that paired question: entries an exact two-sided McNemar test
+ cannot separate (α = 0.05) from the group's anchor share its
+ number, group 1 being anchored on
+ ${stats ? escapeAttr(stats.anchor) : ''}, the highest entry here that
+ publishes per-instance results. It is a property of that comparison, not of an
+ entry on its own. No multiplicity correction is applied, so the groups are if
+ anything smaller than a corrected analysis would make them.
+ Neither number is a repeatability estimate: both condition on the single run
+ each entry reports, and a rerun of the same system is not observable here, so
+ neither is the uncertainty of the ranking.
+ Computed over the ${stats ? stats.with_per_instance : 0} of
+ ${stats ? stats.entries : 0} entries on this board that publish per-instance
+ results. A dash means an entry publishes none, so neither number exists for
+ it — it does not mean the entry stands alone.
+