diff --git a/excore/cli.py b/excore/cli.py index 7ca736b..a053378 100644 --- a/excore/cli.py +++ b/excore/cli.py @@ -41,6 +41,13 @@ def cmd_check(args) -> int: print(f"units {r.units} formats: " + ", ".join(f"{k} x{v}" for k, v in r.formats.items())) print(f"size ~{r.file_gib:.1f} GiB file, ~{r.est_ram_gib:.1f} GiB RAM (estimate), " f"{r.avg_bits_per_weight:.2f} bits/weight on searchable units") + if m.rules: + print(f"rules {len(m.rules)} ({r.default_units} units stay on the default)") + for i in r.unused_rules: + rule = m.rules[i] + layers = f" layers {rule.layers[0]}..{rule.layers[-1]}" if rule.layers else "" + print(f"warning: rule {i} (match {rule.match!r}{layers} -> {rule.format}) has no effect: " + "later rules override every unit it matches", file=sys.stderr) if not r.within_ceiling: print(f"over the {r.ceiling_gib:.1f} GiB RAM ceiling: this recipe would fail the RAM gate", file=sys.stderr) return 3 diff --git a/excore/screen.py b/excore/screen.py index c9f286b..5f1de4e 100644 --- a/excore/screen.py +++ b/excore/screen.py @@ -33,6 +33,13 @@ class ScreenReport: embedding_gib: float = 0.0 avg_bits_per_weight: float = 0.0 ceiling_gib: float = 0.0 + rule_units: tuple[int, ...] = () # units each rule still owns once every later rule has been applied + default_units: int = 0 # units left on the manifest default + + @property + def unused_rules(self) -> tuple[int, ...]: + """Indices of rules that end up owning no unit because later rules override all of theirs.""" + return tuple(i for i, n in enumerate(self.rule_units) if n == 0) @property def file_gib(self) -> float: @@ -55,8 +62,15 @@ def screen(manifest: Manifest, spec: ModelSpec, cfg: Mapping) -> ScreenReport: formats: dict[str, int] = {} total_bytes = 0.0 total_params = 0 + rule_units = [0] * len(manifest.rules) + default_units = 0 for u in units: - fmt = resolve_format(assignments[u.name].format) + a = assignments[u.name] + if a.rule < 0: + default_units += 1 + else: + rule_units[a.rule] += 1 + fmt = resolve_format(a.format) formats[fmt.name] = formats.get(fmt.name, 0) + 1 total_bytes += fmt.bytes_for(u.params) total_params += u.params @@ -71,4 +85,6 @@ def screen(manifest: Manifest, spec: ModelSpec, cfg: Mapping) -> ScreenReport: embedding_gib=embedding / GIB, avg_bits_per_weight=total_bytes * 8 / total_params, ceiling_gib=float(cfg["limits"]["max_peak_rss_gib"]), + rule_units=tuple(rule_units), + default_units=default_units, ) diff --git a/tests/test_cli_plots_sources.py b/tests/test_cli_plots_sources.py index e94c621..21da9a1 100644 --- a/tests/test_cli_plots_sources.py +++ b/tests/test_cli_plots_sources.py @@ -68,6 +68,28 @@ def test_check_command_exit_codes(tiny_config, tmp_path, capsys): assert "unknown quantizer" in capsys.readouterr().err +OVERRIDDEN = RTN.format(n="dead") + " - {match: 'L*.mlp', format: Q5_0}\n - {match: 'L*.mlp', format: Q8_0}\n" + + +def test_screen_counts_units_per_rule(tiny_spec, tiny_cfg): + r = screen(parse_manifest(OVERRIDDEN, tiny_spec), tiny_spec, tiny_cfg) + assert r.rule_units[0] == 1 # lm_head + assert r.rule_units[1] == 0 and r.unused_rules == (1,) # fully overridden by rule 2 + assert r.rule_units[2] > 0 + assert sum(r.rule_units) + r.default_units == r.units + clean = screen(parse_manifest(RTN.format(n="clean"), tiny_spec), tiny_spec, tiny_cfg) + assert clean.unused_rules == () + + +def test_check_warns_about_rules_with_no_effect(tiny_config, tmp_path, capsys): + p = tmp_path / "dead.yaml" + p.write_text(OVERRIDDEN) + assert main(["--config", tiny_config, "check", str(p)]) == 0 # a warning, not an error + cap = capsys.readouterr() + assert "the manifest is legal" in cap.out and "units stay on the default" in cap.out + assert "rule 1" in cap.err and "has no effect" in cap.err and "rule 2" not in cap.err + + def test_check_flags_a_recipe_over_the_ram_ceiling(capsys): assert main(["check", str(ROOT / "experiments/base_variants/V0_baseline.yaml")]) == 0 big = ROOT / "tests" / "_big_tmp.yaml"