excore check: report rule coverage and warn about rules with no effect

excore/cli.py

@@ -41,6 +41,13 @@ def cmd_check(args) -> int:
     print(f"units      {r.units}   formats: " + ", ".join(f"{k} x{v}" for k, v in r.formats.items()))
     print(f"size       ~{r.file_gib:.1f} GiB file, ~{r.est_ram_gib:.1f} GiB RAM (estimate), "
           f"{r.avg_bits_per_weight:.2f} bits/weight on searchable units")
+    if m.rules:
+        print(f"rules      {len(m.rules)} ({r.default_units} units stay on the default)")
+        for i in r.unused_rules:
+            rule = m.rules[i]
+            layers = f" layers {rule.layers[0]}..{rule.layers[-1]}" if rule.layers else ""
+            print(f"warning: rule {i} (match {rule.match!r}{layers} -> {rule.format}) has no effect: "
+                  "later rules override every unit it matches", file=sys.stderr)
     if not r.within_ceiling:
         print(f"over the {r.ceiling_gib:.1f} GiB RAM ceiling: this recipe would fail the RAM gate", file=sys.stderr)
         return 3

excore/screen.py

@@ -33,6 +33,13 @@ class ScreenReport:
     embedding_gib: float = 0.0
     avg_bits_per_weight: float = 0.0
     ceiling_gib: float = 0.0
+    rule_units: tuple[int, ...] = ()      # units each rule still owns once every later rule has been applied
+    default_units: int = 0                # units left on the manifest default
+
+    @property
+    def unused_rules(self) -> tuple[int, ...]:
+        """Indices of rules that end up owning no unit because later rules override all of theirs."""
+        return tuple(i for i, n in enumerate(self.rule_units) if n == 0)
 
     @property
     def file_gib(self) -> float:
@@ -55,8 +62,15 @@ def screen(manifest: Manifest, spec: ModelSpec, cfg: Mapping) -> ScreenReport:
     formats: dict[str, int] = {}
     total_bytes = 0.0
     total_params = 0
+    rule_units = [0] * len(manifest.rules)
+    default_units = 0
     for u in units:
-        fmt = resolve_format(assignments[u.name].format)
+        a = assignments[u.name]
+        if a.rule < 0:
+            default_units += 1
+        else:
+            rule_units[a.rule] += 1
+        fmt = resolve_format(a.format)
         formats[fmt.name] = formats.get(fmt.name, 0) + 1
         total_bytes += fmt.bytes_for(u.params)
         total_params += u.params
@@ -71,4 +85,6 @@ def screen(manifest: Manifest, spec: ModelSpec, cfg: Mapping) -> ScreenReport:
         embedding_gib=embedding / GIB,
         avg_bits_per_weight=total_bytes * 8 / total_params,
         ceiling_gib=float(cfg["limits"]["max_peak_rss_gib"]),
+        rule_units=tuple(rule_units),
+        default_units=default_units,
     )

tests/test_cli_plots_sources.py

@@ -68,6 +68,28 @@ def test_check_command_exit_codes(tiny_config, tmp_path, capsys):
     assert "unknown quantizer" in capsys.readouterr().err
 
 
+OVERRIDDEN = RTN.format(n="dead") + "  - {match: 'L*.mlp', format: Q5_0}\n  - {match: 'L*.mlp', format: Q8_0}\n"
+
+
+def test_screen_counts_units_per_rule(tiny_spec, tiny_cfg):
+    r = screen(parse_manifest(OVERRIDDEN, tiny_spec), tiny_spec, tiny_cfg)
+    assert r.rule_units[0] == 1                         # lm_head
+    assert r.rule_units[1] == 0 and r.unused_rules == (1,)   # fully overridden by rule 2
+    assert r.rule_units[2] > 0
+    assert sum(r.rule_units) + r.default_units == r.units
+    clean = screen(parse_manifest(RTN.format(n="clean"), tiny_spec), tiny_spec, tiny_cfg)
+    assert clean.unused_rules == ()
+
+
+def test_check_warns_about_rules_with_no_effect(tiny_config, tmp_path, capsys):
+    p = tmp_path / "dead.yaml"
+    p.write_text(OVERRIDDEN)
+    assert main(["--config", tiny_config, "check", str(p)]) == 0          # a warning, not an error
+    cap = capsys.readouterr()
+    assert "the manifest is legal" in cap.out and "units stay on the default" in cap.out
+    assert "rule 1" in cap.err and "has no effect" in cap.err and "rule 2" not in cap.err
+
+
 def test_check_flags_a_recipe_over_the_ram_ceiling(capsys):
     assert main(["check", str(ROOT / "experiments/base_variants/V0_baseline.yaml")]) == 0
     big = ROOT / "tests" / "_big_tmp.yaml"