Leaderboard: sampling error and McNemar significance groups

build.py

@@ -2,6 +2,7 @@
 import json
 import pathlib
 import shutil
+import sys
 from flask import Flask
 from flask_flatpages import FlatPages
 from jinja2 import Environment, FileSystemLoader, select_autoescape
@@ -11,6 +12,9 @@
 TEMPLATES = ROOT / "templates"
 DIST = ROOT / "dist"
 
+sys.path.insert(0, str(ROOT / "data"))
+from leaderboard_statistics import annotate_all  # noqa: E402
+
 def get_pages():
     pages = {}
     pages_dir = TEMPLATES / "pages"
@@ -126,6 +130,31 @@ def main() -> None:
     # load data
     with open(ROOT / "data/leaderboards.json", "r") as f:
         leaderboards = json.load(f)
+
+    # Sampling error and significance groups, computed here rather than stored,
+    # so that leaderboards.json stays the record of what was submitted and these
+    # stay a function of it. Boards without per-instance results are untouched.
+    for summary in annotate_all(
+        leaderboards["leaderboards"] if isinstance(leaderboards, dict) else leaderboards
+    ):
+        line = (f"stats: {summary['name']}: {summary['with_per_instance']}/{summary['entries']} "
+                f"entries with per-instance results")
+        if summary["groups"]:
+            line += (f", {summary['groups']} significance groups, "
+                     f"{summary['top_tie_group_size']} in group 1 anchored on "
+                     f"{summary['anchor']}")
+            if summary["unmeasured_above_anchor"]:
+                line += f" ({summary['unmeasured_above_anchor']} higher entries publish none)"
+        if summary["not_comparable"]:
+            line += (f", {summary['not_comparable']} entries share no instance with the "
+                     "anchor of their group")
+        print(line)
+        # An entry dropped for contradicting its own published rate is a defect
+        # in the submitted data, not a quiet gap in a column, so the build says
+        # so by name rather than leaving a dash on the page as the only trace.
+        for name, reason in summary["excluded"]:
+            print(f"stats: {summary['name']}: {name}: {reason}")
+
     with open(ROOT / "data/press.json", "r") as f:
         press = json.load(f)
         press = sorted(press, key=lambda x: x["date"], reverse=True)

css/leaderboard.css

@@ -782,10 +782,49 @@
   min-width: 66rem;
 }
 
+/* The two statistics columns add sized width, so the point at which the wrapper
+   starts scrolling instead of squeezing Model moves with them. */
+#leaderboard-container .data-table[data-stats="true"] {
+  min-width: 74rem;
+}
+
 #leaderboard-container col.cw-select { width: 2rem; }
 #leaderboard-container col.cw-rank { width: 3.1rem; }
 #leaderboard-container col.cw-agent { width: 12rem; }
 #leaderboard-container col.cw-resolved { width: 7.25rem; }
+/* Statistics columns: the error on one entry, and the paired group it sits in */
+#leaderboard-container .stat-cell .number,
+#leaderboard-container .stat-cell .text-muted {
+  font-variant-numeric: tabular-nums;
+}
+
+#leaderboard-container .tie-group {
+  display: inline-block;
+  min-width: 1.5rem;
+  padding: 0.05rem 0.35rem;
+  border-radius: var(--radius-sm, 0.25rem);
+  font-variant-numeric: tabular-nums;
+  color: var(--color-text-muted);
+}
+
+/* Group 1 is the one the headline number is read off, so it is the one worth
+   seeing at a glance. Emphasis, not a claim of quality. The tint is the same
+   accent the site already defines for both themes, so it stays visible in dark
+   mode instead of disappearing into the background. */
+#leaderboard-container .tie-group-top {
+  background-color: var(--color-accent-light);
+  color: var(--color-text);
+  font-weight: 600;
+}
+
+.stats-note {
+  margin: 0.5rem 0 0;
+  font-size: 0.8125rem;
+  line-height: 1.5;
+}
+
+#leaderboard-container col.cw-se { width: 4.75rem; }
+#leaderboard-container col.cw-tie { width: 3.5rem; }
 #leaderboard-container col.cw-cost { width: 5.25rem; }
 #leaderboard-container col.cw-trajs { width: 4.5rem; }
 #leaderboard-container col.cw-org { width: 4.75rem; }