From 0b8b4bd7dac29e1df3b538342d13c9fc14625669 Mon Sep 17 00:00:00 2001 From: vu1n <2105991+vu1n@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:31:20 +0000 Subject: [PATCH 01/43] distill: mine "A fails here, B works" repair pitfalls from shell runs --- docs/features/distill.md | 11 ++ kypp/distill.py | 29 ++++- kypp/repairs.py | 232 +++++++++++++++++++++++++++++++++++++++ tests/test_repairs.py | 122 ++++++++++++++++++++ 4 files changed, 389 insertions(+), 5 deletions(-) create mode 100644 kypp/repairs.py create mode 100644 tests/test_repairs.py diff --git a/docs/features/distill.md b/docs/features/distill.md index fcd689d..02f4f30 100644 --- a/docs/features/distill.md +++ b/docs/features/distill.md @@ -4,6 +4,7 @@ paths: - "kypp/distill.py" - "kypp/batch_distill.py" - "kypp/episodes.py" + - "kypp/repairs.py" - "scripts/eval_episodes.py" - "eval/episodes/*" - "docs/episode-extraction-evaluation.md" @@ -12,6 +13,7 @@ entry: - "kypp/batch_distill.py" tests: - "tests/test_episodes.py" + - "tests/test_repairs.py" --- # Distillation @@ -20,6 +22,15 @@ distiller runs when `KYPP_DISTILL_MODEL` is set (`claude`, `codex` or an ollama ## Gotchas - Without `KYPP_DISTILL_MODEL` you only get heuristic, failure-mined pitfalls. +- Repair pitfalls (`kypp/repairs.py`) run with or without a model: a shell command that fails + with command-not-found, a missing module or a missing env var, followed in the same session by a + form that works (`uv run pytest`, `.venv/bin/python`, `V=x cmd`). The subject names only what + failed, so a second session's repair lands on the same subject and the K=2 gate accepts it; a + one-session quirk stays a candidate. A repair suppresses the generic "`` repeatedly + failed" pitfall for the failures it explains. +- A repair is skipped when the bare form works later in the session, when the thing was installed + in between, or when the working form is a host path. A secret-looking env var's value is never + written. - If the LLM distiller fails, it falls back to the heuristic distiller rather than raising. - With `TYPESAFE_API_KEY` or `KYPP_DECIDER_URL` (a local strands-decider) set, and `kypp[s1]` installed, a System One gate drops drafts scoring diff --git a/kypp/distill.py b/kypp/distill.py index c058c61..0765110 100644 --- a/kypp/distill.py +++ b/kypp/distill.py @@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Protocol from . import episodes as _episodes -from . import s1 +from . import repairs, s1 from .vocab import OLLAMA_DEFAULT_HOST, TYPES # single-sourced vocabulary (turso-free leaf) if TYPE_CHECKING: @@ -169,14 +169,32 @@ def distill(self, trace: Trace) -> list[ClaimDraft]: f"in this session's observations", 0.6)) if trace.run_failed: drafts.append(ClaimDraft("pitfall", "run failed", _trunc(trace.run_failed), 0.5)) + fixes, explained = repairs.mine(trace) for name, n in trace.tool_failures.items(): - if n >= 2: - sample = next(a for a in trace.actions if a.failed and a.name == name) + unexplained = [a for i, a in enumerate(trace.actions) + if a.failed and a.name == name and i not in explained] + # A failure a repair explains is said better by the repair ("A fails, B works"). + if n >= 2 and unexplained: + sample = unexplained[0] drafts.append(ClaimDraft( "pitfall", f"{name} repeatedly failed", _trunc(f"`{name}` errored {n}x this run; e.g. {sample.output}"), 0.4, code_refs=_refs_from(sample.input))) - return drafts + return drafts + fixes + + +class WithRepairs: + """Add the deterministic repair pitfalls to another distiller's drafts. The LLM distiller is told + to skip environment artifacts; a repair is one only until a second session hits it, and then + the K=2 gate accepts it.""" + + def __init__(self, inner: Distiller): + self.inner = inner + + def distill(self, trace: Trace) -> list[ClaimDraft]: + drafts = self.inner.distill(trace) + taken = {d.subject for d in drafts} + return drafts + [d for d in repairs.mine(trace)[0] if d.subject not in taken] # LLM_DISTILL_PROMPT + _SCHEMA are what the model is ASKED for (model-agnostic, code-grounded claims). @@ -404,7 +422,8 @@ def distiller_from_env() -> Distiller: A configured System One model (see kypp.s1) adds the write gate on top. The seam-config analog of store.store_from_env.""" model = os.environ.get("KYPP_DISTILL_MODEL") - inner = (FallbackDistiller(LLMDistiller(_complete_from_model(model), episodes=_episodes.extractor_from_env()), + inner = (FallbackDistiller(WithRepairs(LLMDistiller(_complete_from_model(model), + episodes=_episodes.extractor_from_env())), HeuristicDistiller()) if model else HeuristicDistiller()) via = s1.client() diff --git a/kypp/repairs.py b/kypp/repairs.py new file mode 100644 index 0000000..bd7415a --- /dev/null +++ b/kypp/repairs.py @@ -0,0 +1,232 @@ +"""repairs.py — §0 trace → "A fails here, B works" pitfalls, mined without a model. + +An agent often rediscovers the same environment fact by failing at it: `pytest` is not on PATH but +`uv run pytest` works, `python` can't import the project but `.venv/bin/python` can, a script needs +`PYTHONPATH=src`. Within one session this mines a failing shell command followed by a later one that +does the same thing another way and works, and states both forms. The subject names only what failed +(never the fix), so the same fact from a second session reuses the subject and the arbiter's K=2 +gate accepts it: one session's quirk stays a candidate, a recurring one becomes memory. + +Idea borrowed from TokenEconomizer's environment facts (microsoft/TokenEconomizer, Apache-2.0): +its FormRepair and EnvValue families. Re-implemented here over §0 actions. +""" +from __future__ import annotations + +import re +import shlex +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .distill import Action, ClaimDraft, Trace + +# Shell tools as each harness names them (Claude: Bash; Codex: shell / exec_command; pillbox: bash). +_SHELL_TOOLS = {"bash", "shell", "exec_command", "local_shell", "run_shell_command"} + +_SEGMENT_SPLIT = re.compile(r"\s*(?:&&|\|\||;)\s*") # a pipe stays inside its segment +_SETUP = {"cd", "source", ".", "export", "set", "pushd", "unset"} +_ASSIGN = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=(.*)$", re.S) +_WORD = re.compile(r"^[A-Za-z0-9._+-]{1,64}$") # a command or module name we're willing to quote +# A missing command, as bash, sh, zsh and env each word it. +_CMD_MISSING = [re.compile(p) for p in ( + r"(?:^|\s|:)([A-Za-z0-9._+-]+): (?:command )?not found", + r"command not found: ([A-Za-z0-9._+-]+)", + r"env: '?([A-Za-z0-9._+-]+)'?: No such file or directory", +)] +_MOD_MISSING = [re.compile(p) for p in ( + r"No module named '?([A-Za-z0-9_.]+)'?", + r"Cannot find module '([^']+)'", +)] +_ENV_MISSING = [re.compile(p) for p in ( + r"KeyError: '([A-Z][A-Z0-9_]+)'", + r"\b([A-Z][A-Z0-9_]{2,})\b (?:environment variable )?(?:is )?(?:not set|unset|required|missing)", + r"(?:environment variable|env var)s? '?([A-Z][A-Z0-9_]{2,})'? (?:is )?(?:not set|unset|required|missing)", + r"[Mm]issing (?:required )?(?:environment variable|env var) '?([A-Z][A-Z0-9_]{2,})'?", +)] +# A later command led by one of these is looking or installing, not running the thing another way. +_NOT_A_REPAIR = {"which", "type", "command", "whereis", "hash", "echo", "printf", "ls", "cat", "man", + "file", "stat", "find", "fd", "rg", "grep", "sudo", "apt", "apt-get", "brew", "pip", + "pip3", "pipx", "conda", "cargo", "go", "gem", "npm", "pnpm", "yarn"} +_INSTALL_WORDS = {"install", "add", "i", "sync", "-v", "--version", "--help", "-h"} +# A value under one of these names may be a credential: name the variable, never repeat its value. +_SECRET_NAME = re.compile(r"KEY|TOKEN|SECRET|PASS|AUTH|CRED|PRIVATE|COOKIE|SESSION", re.I) +_EXIT_CODE = re.compile(r"(?:Exit code:?|\"exit_code\":)\s*(-?\d+)") +_ABS = re.compile(r"(?:^|\s)/(?:var|tmp|private|Users|home|root)/") + + +@dataclass +class _Run: + i: int + command: str + segments: list[list[str]] # each shell segment, tokenized, env assignments kept as leading tokens + output: str + failed: bool + + +def _command_of(a: Action) -> str | None: + if (a.name or "").lower() not in _SHELL_TOOLS or not isinstance(a.input, dict): + return None + c = a.input.get("command", a.input.get("cmd")) + if isinstance(c, list): # codex: ["bash", "-lc", "