diff --git a/docs/features/distill.md b/docs/features/distill.md index fcd689d..f901b10 100644 --- a/docs/features/distill.md +++ b/docs/features/distill.md @@ -4,6 +4,7 @@ paths: - "kypp/distill.py" - "kypp/batch_distill.py" - "kypp/episodes.py" + - "kypp/repairs.py" - "scripts/eval_episodes.py" - "eval/episodes/*" - "docs/episode-extraction-evaluation.md" @@ -12,6 +13,7 @@ entry: - "kypp/batch_distill.py" tests: - "tests/test_episodes.py" + - "tests/test_repairs.py" --- # Distillation @@ -20,6 +22,22 @@ distiller runs when `KYPP_DISTILL_MODEL` is set (`claude`, `codex` or an ollama ## Gotchas - Without `KYPP_DISTILL_MODEL` you only get heuristic, failure-mined pitfalls. +- Repair pitfalls (`kypp/repairs.py`) run with or without a model: a shell command that fails + with command-not-found, a missing module or a missing env var, followed in the same session by a + form that works (`uv run pytest`, `.venv/bin/python`, `V=x cmd`). The subject names only what + failed, so a second session's repair lands on the same subject and the K=2 gate accepts it; a + one-session quirk stays a candidate. A repair suppresses the generic "`` repeatedly + failed" pitfall for the failures it explains. +- A repair needs a later run with a confirmed success (Claude's `completed`, Codex's exit 0) that + runs the same thing: the same script or `-m module` for a missing module or env var (inline + `-c` code never counts). Only a segment followed by nothing but `&&`, outside any `||` or pipe, + counts as having succeeded, since a chain exits with its last command. It is + skipped when the original works later without the fix, when the thing was installed in between, + when the later run only probes (`--version`, `--help`), or when the working form is a host path. +- Values are written only for a short allowlist of env vars (`PYTHONPATH`, `NODE_ENV`, ...), and + never one with `@` in it. A fix runs the command by path or behind a known wrapper (`_WRAPPERS`: + `uv run`, `python -m`, `npx`, ...). Nothing else counts, so `touch pytest` isn't a fix, and flags + and their values (`--token X`, `-e K=V`) never reach a claim. - If the LLM distiller fails, it falls back to the heuristic distiller rather than raising. - With `TYPESAFE_API_KEY` or `KYPP_DECIDER_URL` (a local strands-decider) set, and `kypp[s1]` installed, a System One gate drops drafts scoring diff --git a/kypp/distill.py b/kypp/distill.py index c058c61..0765110 100644 --- a/kypp/distill.py +++ b/kypp/distill.py @@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Protocol from . import episodes as _episodes -from . import s1 +from . import repairs, s1 from .vocab import OLLAMA_DEFAULT_HOST, TYPES # single-sourced vocabulary (turso-free leaf) if TYPE_CHECKING: @@ -169,14 +169,32 @@ def distill(self, trace: Trace) -> list[ClaimDraft]: f"in this session's observations", 0.6)) if trace.run_failed: drafts.append(ClaimDraft("pitfall", "run failed", _trunc(trace.run_failed), 0.5)) + fixes, explained = repairs.mine(trace) for name, n in trace.tool_failures.items(): - if n >= 2: - sample = next(a for a in trace.actions if a.failed and a.name == name) + unexplained = [a for i, a in enumerate(trace.actions) + if a.failed and a.name == name and i not in explained] + # A failure a repair explains is said better by the repair ("A fails, B works"). + if n >= 2 and unexplained: + sample = unexplained[0] drafts.append(ClaimDraft( "pitfall", f"{name} repeatedly failed", _trunc(f"`{name}` errored {n}x this run; e.g. {sample.output}"), 0.4, code_refs=_refs_from(sample.input))) - return drafts + return drafts + fixes + + +class WithRepairs: + """Add the deterministic repair pitfalls to another distiller's drafts. The LLM distiller is told + to skip environment artifacts; a repair is one only until a second session hits it, and then + the K=2 gate accepts it.""" + + def __init__(self, inner: Distiller): + self.inner = inner + + def distill(self, trace: Trace) -> list[ClaimDraft]: + drafts = self.inner.distill(trace) + taken = {d.subject for d in drafts} + return drafts + [d for d in repairs.mine(trace)[0] if d.subject not in taken] # LLM_DISTILL_PROMPT + _SCHEMA are what the model is ASKED for (model-agnostic, code-grounded claims). @@ -404,7 +422,8 @@ def distiller_from_env() -> Distiller: A configured System One model (see kypp.s1) adds the write gate on top. The seam-config analog of store.store_from_env.""" model = os.environ.get("KYPP_DISTILL_MODEL") - inner = (FallbackDistiller(LLMDistiller(_complete_from_model(model), episodes=_episodes.extractor_from_env()), + inner = (FallbackDistiller(WithRepairs(LLMDistiller(_complete_from_model(model), + episodes=_episodes.extractor_from_env())), HeuristicDistiller()) if model else HeuristicDistiller()) via = s1.client() diff --git a/kypp/repairs.py b/kypp/repairs.py new file mode 100644 index 0000000..2c87f0d --- /dev/null +++ b/kypp/repairs.py @@ -0,0 +1,474 @@ +"""repairs.py — §0 trace → "A fails here, B works" pitfalls, mined without a model. + +An agent often rediscovers the same environment fact by failing at it: `pytest` is not on PATH but +`uv run pytest` works, `python` can't import the project but `.venv/bin/python` can, a script needs +`PYTHONPATH=src`. Within one session this mines a failing shell command followed by a later one that +does the same thing another way and works, and states both forms. The subject names only what failed +(never the fix), so the same fact from a second session reuses the subject and the arbiter's K=2 +gate accepts it: one session's quirk stays a candidate, a recurring one becomes memory. + +Idea borrowed from TokenEconomizer's environment facts (microsoft/TokenEconomizer, Apache-2.0): +its FormRepair and EnvValue families. Re-implemented here over §0 actions. +""" +from __future__ import annotations + +import re +import shlex +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .distill import Action, ClaimDraft, Trace + +# Shell tools as each harness names them (Claude: Bash; Codex: shell / exec_command; pillbox: bash). +_SHELL_TOOLS = {"bash", "shell", "exec_command", "local_shell", "run_shell_command"} + +_OPS = {";", "&&", "||", "|", "&"} +_SETUP = {"cd", "source", ".", "export", "set", "pushd", "unset", ":"} +_ASSIGN = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*)=(.*)$", re.S) +_WORD = re.compile(r"^[A-Za-z0-9._+-]{1,64}$") # a command or module name we're willing to quote +# A missing command, as bash, sh, zsh and env each word it. +_CMD_MISSING = [re.compile(p) for p in ( + r"(?:^|\s|:)([A-Za-z0-9._+-]+): (?:command )?not found", + r"command not found: ([A-Za-z0-9._+-]+)", + r"env: '?([A-Za-z0-9._+-]+)'?: No such file or directory", +)] +_MOD_MISSING = [re.compile(p) for p in ( + r"No module named '?([A-Za-z0-9_.]+)'?", + r"Cannot find module '([^']+)'", +)] +_ENV_MISSING = [re.compile(p) for p in ( + r"KeyError: '([A-Z][A-Z0-9_]+)'", + r"\b([A-Z][A-Z0-9_]{2,})\b (?:environment variable )?(?:is )?(?:not set|unset|required|missing)", + r"(?:environment variable|env var)s? '?([A-Z][A-Z0-9_]{2,})'? (?:is )?(?:not set|unset|required|missing)", + r"[Mm]issing (?:required )?(?:environment variable|env var) '?([A-Z][A-Z0-9_]{2,})'?", +)] +# The wrappers a fix may run the command behind (or none, with the command given by path). Any +# other prefix (`which`, `touch`, `test -x`, `docker run -e K=V`) doesn't run it, or may carry a +# credential. +_WRAPPERS = {("uv", "run"), ("uvx",), ("poetry", "run"), ("pipenv", "run"), ("pdm", "run"), + ("hatch", "run"), ("rye", "run"), ("python", "-m"), ("python3", "-m"), ("npx",), + ("bunx",), ("bun", "x"), ("bun", "run"), ("pnpm", "exec"), ("pnpm", "dlx"), + ("npm", "exec"), ("yarn",), ("yarn", "exec"), ("bundle", "exec")} +_INSTALLS = {"install", "add", "i", "sync"} # a run with one of these may have installed anything +# Arguments that only ask the command about itself: running it this way proves nothing works. +_PROBE_ARGS = {"--version", "-V", "version", "--help", "-h", "help"} +_PROBE_FLAGS = {"--version", "-V", "--help", "-h"} # with any other arguments it still only probes +# Only these variables' values are written; any other value may be a credential (a DATABASE_URL +# carries a password), so a fact names the variable and never repeats its value. +_SAFE_ENV = {"PYTHONPATH", "NODE_ENV", "CI", "TZ", "LANG", "LC_ALL", "RUST_BACKTRACE", "RUST_LOG", + "DEBUG", "PORT", "HOST", "FORCE_COLOR", "NO_COLOR", "PYTHONUNBUFFERED", "PYTHONDONTWRITEBYTECODE"} +_USERINFO = re.compile(r"://[^/\s]*@") +_EXIT_CODE = re.compile(r"(?:Exit code:?|\"exit_code\":|Process exited with code)\s*(-?\d+)") +# Any absolute or home-relative path is this host's, wherever it is rooted. +_ABS = re.compile(r"(?:^|[\s:=])[/~]") + + +@dataclass +class _Run: + i: int + command: str + segments: list[list[str]] # each shell segment, tokenized, env assignments kept as leading tokens + proven: list[list[str]] # the segments the run's success proves ran and succeeded + output: str + ok: bool # confirmed to have succeeded; a run with no recorded outcome is neither + may_fail: bool # failed, or its exit status could hide a failure (`pytest | tail`, `pytest; echo`) + where: tuple | None # where the shell was when it ran; None when a `cd` follows a command, so the place is unclear + installs: bool # runs an install or sync + sets_env: bool # in a persistent shell, exports, unsets or sources something, or assigns on its own + reads_input: bool # a `<` input may choose what the script does, so two runs of it can differ + + +def _command_of(a: Action) -> str | None: + if (a.name or "").lower() not in _SHELL_TOOLS or not isinstance(a.input, dict): + return None + c = a.input.get("command", a.input.get("cmd")) + if isinstance(c, list): # codex: ["bash", "-lc", "