diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..749c9593 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,18 @@ +# Knowledge maintainers own the knowledge gate: its verifier bundle, workflows, +# signing key and this file, plus the knowledge tooling and the non-governed +# knowledge paths. With the ruleset's "require review from Code Owners" and +# "dismiss stale approvals", a change here cannot land without a maintainer +# approving its current head (doc/features/knowledge-service.md). +# +# Governed knowledge pages (knowledge/repos/**, knowledge/general/**) have NO +# code owner on purpose: they merge through the kb-gate signed verdict. A +# human-approved verdict must be approved by an owner of /.github/kb-gate/. +# +# GitHub does not let an author approve their own PR, and knowledge PRs are +# opened as @tzhouam, so at least one more maintainer must be listed below +# before the human-approved path can be used (owner setup step). + +/.github/ @tzhouam +/knowledge/tools/ @tzhouam +/knowledge/skills/ @tzhouam +/knowledge/.claude/ @tzhouam diff --git a/.github/kb-gate/MANIFEST.sha256 b/.github/kb-gate/MANIFEST.sha256 new file mode 100644 index 00000000..88a019f3 --- /dev/null +++ b/.github/kb-gate/MANIFEST.sha256 @@ -0,0 +1,9 @@ +5f6e269fbbb951f9a49c40474a3d71c556ac389f5ebd5545f0ac762f85361723 README.md +77d88b9671927ab95bc6ac9ed57aaa24226423e151843381c2a3fa90cd0cafde config.json +e8157d90270e9f437b364e9ef95bc0603c80c6421fa591953525387cc4f5d4e0 kb_gate.py +5743c4a6943c291073c5344d408eaec28e8bc7440a8ee113365588728ded5e8a l1.py +39f69aa816b83b673443601d38e57040521664c5aca407154ba40a8a35bc61d9 lifecycle.py +b0d0fab0210870f989da80f7ffba3978dbf723f4f087025524d1963c4081a009 ops.py +67f2ebd85e4e886f3facc35a181f575fd6291f1e4a4068f5daca99191609aace requirements.lock +e7fbe6f3d6b02fadfbd86e32e67d588ae698ae2de19ac91aa5db2a250cb93946 signing.py +5c5015e7e9e6314f84e0337a58d9aa1bb6f7ec21ec830e4907dc5c35bbca4a09 verdict.py diff --git a/.github/kb-gate/README.md b/.github/kb-gate/README.md new file mode 100644 index 00000000..cb74bce2 --- /dev/null +++ b/.github/kb-gate/README.md @@ -0,0 +1,23 @@ +# kb-gate verifier bundle + +The required check `kb-gate` runs **only** from this directory. It is generated by +`tools/build_kb_gate_bundle.py` from `src/infermatrix_copilot/knowledge_service/` +(`gate_verifier.py` → `kb_gate.py`, plus `l1`, `lifecycle`, `ops`, `signing`, +`verdict`); do not edit the `.py` files here by hand. The job installs +`requirements.lock` with `--require-hashes --no-deps` and checks `MANIFEST.sha256` +before running anything. + +Changing anything here is a `.github/**` change and needs CODEOWNERS approval, so a +code PR that weakens `src/` or `knowledge/tools/` cannot weaken the gate. + +`config.json`: + +- `holds_url`: the public HTTPS URL of the knowledge service's signed hold list + (`/public/holds.json` on the bot host). Until it is set, every PR that + touches `knowledge/` fails closed. PRs that do not touch `knowledge/` always pass. + +The service's public key is `.github/kb-gate.pub` (`ed25519 `, the output of +`kb keygen`). Without it, every knowledge PR fails closed. + +See `doc/features/knowledge-service.md` for the stages (PR precheck and merge group) +and what each checks. diff --git a/.github/kb-gate/config.json b/.github/kb-gate/config.json new file mode 100644 index 00000000..a497977c --- /dev/null +++ b/.github/kb-gate/config.json @@ -0,0 +1,3 @@ +{ + "holds_url": "" +} diff --git a/.github/kb-gate/kb_gate.py b/.github/kb-gate/kb_gate.py new file mode 100644 index 00000000..417ca858 --- /dev/null +++ b/.github/kb-gate/kb_gate.py @@ -0,0 +1,692 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/gate_verifier.py -- do not edit here. +"""kb-gate: the repository-side verifier for knowledge pull requests. + +It is the required check ``kb-gate`` and runs in two places under one name: + +* PR stage (``pull_request_target`` / ``issue_comment`` / dispatch): verifies + the PR against the current base and posts a ``kb-gate`` commit status on the + PR head. It only decides whether a PR may ENTER the merge queue. +* merge group stage (``merge_group``): verifies every segment of the group on + the exact commit that would land and exits non-zero on any failure. This is + what decides whether anything lands. + +A change that does not touch ``knowledge/`` passes. A knowledge change passes +only when all of these hold for it: + +1. L1: whitelisted paths only, regular files only, no new tree issues, on the + PR's own change AND on the tree that would land; +2. a valid Ed25519-signed verdict comment for this repository, PR and exact + head SHA, inside its issue window (the comment's author is irrelevant); +3. the signed patch manifest equals the change as applied (base independent); +4. ``auto``: the signed block table equals the blocks L1 derives, every block + passed, the consistency judgements cover the same owner directories and + their page hashes match the tree that would land; no page in the context + set changed between the judged base and the effective base; + ``human-approved``: every bound review is still APPROVED on this head, one + is by a knowledge maintainer (a CODEOWNERS owner of ``.github/kb-gate/``), + and ``kb:human-approved`` is set; +5. no ``kb:hold`` label and not held by the service's signed hold list (a + missing, forged or stale hold list fails closed). + +PR content is only ever READ through git objects; nothing from the PR is +executed or installed. This module is vendored into ``.github/kb-gate/`` by +``tools/build_kb_gate_bundle.py`` and runs from there, so it imports nothing +outside the standard library and its vendored siblings. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Iterable, Mapping + +try: + from .l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree + from .lifecycle import LifecycleError, Page + from .ops import all_rule_ids, repo_scope + from .signing import SignatureError, load_public_key, verify + from .verdict import VerdictError, check_binding, manifests_equal +except ImportError: # pragma: no cover - vendored verifier bundle + from l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree # type: ignore[no-redef] + from lifecycle import LifecycleError, Page # type: ignore[no-redef] + from ops import all_rule_ids, repo_scope # type: ignore[no-redef] + from signing import SignatureError, load_public_key, verify # type: ignore[no-redef] + from verdict import VerdictError, check_binding, manifests_equal # type: ignore[no-redef] + +STATUS_CONTEXT = "kb-gate" +VERDICT_MARKER = "" +HOLD_LABEL = "kb:hold" +HUMAN_LABEL = "kb:human-approved" +HOLDS_MAX_AGE = 10 * 60 +MAINTAINER_PATH = ".github/kb-gate/kb_gate.py" +WHITELIST_CODES = ("path_not_whitelisted", "mode_not_regular") +KNOWLEDGE_PREFIX = "knowledge/" +GOVERNED_PREFIXES = ("knowledge/repos/", "knowledge/general/") +GOVERNED_SUFFIXES = (".md", ".yaml") +EXTERNAL_REF_DIRS = ("skills/", "plugins/", "adapters/", "doc/", "playbooks/") +EXTERNAL_SUFFIXES = (".md", ".yaml", ".yml", ".json", ".txt") +_ZERO = "0" * 40 + + +class GateError(RuntimeError): + """The verifier could not establish a fact it needs (fails closed).""" + + +# -- git (objects only; nothing from the PR is ever checked out or executed) --- + +class Git: + def __init__(self, cwd: str | Path): + self.cwd = str(cwd) + + def run(self, *args: str, input: bytes | None = None, ok: tuple[int, ...] = (0,)) -> bytes: + proc = subprocess.run(["git", *args], cwd=self.cwd, input=input, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode not in ok: + raise GateError(f"git {' '.join(args)} failed: {proc.stderr.decode(errors='replace')[:400]}") + return proc.stdout + + def rev(self, name: str) -> str: + return self.run("rev-parse", "--verify", "--quiet", f"{name}^{{commit}}").decode().strip() + + def tree_of(self, name: str) -> str: + return self.run("rev-parse", "--verify", f"{name}^{{tree}}").decode().strip() + + def parents(self, sha: str) -> list[str]: + return self.run("rev-list", "--parents", "-n", "1", sha).decode().split()[1:] + + def is_ancestor(self, older: str, newer: str) -> bool: + proc = subprocess.run(["git", "merge-base", "--is-ancestor", older, newer], cwd=self.cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=False) + if proc.returncode not in (0, 1): + raise GateError(f"cannot relate {older[:12]} and {newer[:12]}: " + f"{proc.stderr.decode(errors='replace')[:200]}") + return proc.returncode == 0 + + def merge_base(self, a: str, b: str) -> str: + return self.run("merge-base", a, b).decode().strip() + + def first_parent_chain(self, base: str, head: str) -> list[str]: + return self.run("rev-list", "--first-parent", "--reverse", f"{base}..{head}").decode().split() + + def merge_tree(self, base: str, head: str) -> str: + """The tree a merge of ``head`` into ``base`` would produce (PR stage).""" + proc = subprocess.run(["git", "merge-tree", "--write-tree", base, head], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode != 0 and b"--write-tree" in proc.stderr: # git < 2.38: no --write-tree + return self._index_merge(base, head) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().split()[0] + + def _index_merge(self, base: str, head: str) -> str: + """Only merges where each path changed on at most one side; anything + else is reported as not merging cleanly (stricter, never looser).""" + import tempfile + + with tempfile.TemporaryDirectory() as scratch: + env = {**os.environ, "GIT_INDEX_FILE": str(Path(scratch) / "index")} + for args in (["read-tree", "-i", "-m", self.merge_base(base, head), base, head], ["write-tree"]): + proc = subprocess.run(["git", *args], cwd=self.cwd, env=env, check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().strip() + + def raw_diff(self, old: str, new: str) -> list[dict]: + """Every changed path as a manifest entry (renames are D + A).""" + out = self.run("diff", "--raw", "-z", "--no-renames", "--no-abbrev", old, new).split(b"\0") + entries: list[dict] = [] + index = 0 + while index < len(out) - 1: + meta = out[index].decode() + path = out[index + 1].decode("utf-8", "replace") + index += 2 + if not meta.startswith(":"): + raise GateError(f"unexpected git diff output: {meta!r}") + old_mode, new_mode, old_blob, new_blob, status = meta[1:].split() + entries.append({ + "path": path, "status": status[0], + "old_blob": "" if old_blob == _ZERO else old_blob, + "new_blob": "" if new_blob == _ZERO else new_blob, + "old_mode": "" if old_mode == "000000" else old_mode, + "new_mode": "" if new_mode == "000000" else new_mode, + }) + return entries + + def changed_names(self, old: str, new: str) -> list[str]: + out = self.run("diff", "--name-only", "-z", "--no-renames", old, new, "--", "knowledge/") + return [name.decode("utf-8", "replace") for name in out.split(b"\0") if name] + + def texts(self, rev: str, prefixes: tuple[str, ...], suffixes: tuple[str, ...]) -> dict[str, str]: + listing = self.run("ls-tree", "-r", "-z", "--full-tree", rev).split(b"\0") + wanted: list[tuple[str, str]] = [] + for entry in listing: + if not entry: + continue + meta, _, raw_path = entry.partition(b"\t") + mode, kind, oid = meta.decode().split() + name = raw_path.decode("utf-8", "replace") + if kind == "blob" and mode == "100644" and name.startswith(prefixes) and name.endswith(suffixes): + wanted.append((name, oid)) + if not wanted: + return {} + out = self.run("cat-file", "--batch", input="".join(f"{oid}\n" for _, oid in wanted).encode()) + texts: dict[str, str] = {} + cursor = 0 + for name, _oid in wanted: + header_end = out.index(b"\n", cursor) + size = int(out[cursor:header_end].split()[2]) + body = out[header_end + 1: header_end + 1 + size] + cursor = header_end + 1 + size + 1 + try: + texts[name] = body.decode("utf-8") + except UnicodeDecodeError: + continue + return texts + + def knowledge_files(self, rev: str) -> dict[str, str]: + files = self.texts(rev, GOVERNED_PREFIXES, GOVERNED_SUFFIXES) + return {name[len(KNOWLEDGE_PREFIX):]: text for name, text in files.items()} + + def external_texts(self, rev: str) -> dict[str, str]: + return self.texts(rev, EXTERNAL_REF_DIRS, EXTERNAL_SUFFIXES) + + def show(self, rev: str, path: str) -> str | None: + proc = subprocess.run(["git", "show", f"{rev}:{path}"], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + return proc.stdout.decode("utf-8", "replace") if proc.returncode == 0 else None + + +# -- GitHub REST (read, plus the PR-stage status) -------------------------------- + +class GitHub: + def __init__(self, repository: str, token: str, api: str = "https://api.github.com"): + self.repository, self._token, self._api = repository, token, api.rstrip("/") + + def _request(self, method: str, path: str, body: dict | None = None) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(self._api + path, data=data, method=method, headers={ + "Accept": "application/vnd.github+json", "Authorization": f"Bearer {self._token}", + "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "kb-gate"}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read() or b"null") + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"GitHub {method} {path} failed: {exc}") from exc + + def get(self, path: str) -> Any: + return self._request("GET", path) + + def get_all(self, path: str, *, pages: int = 10) -> list: + out: list = [] + sep = "&" if "?" in path else "?" + for page in range(1, pages + 1): + batch = self.get(f"{path}{sep}per_page=100&page={page}") + out.extend(batch) + if len(batch) < 100: + break + return out + + def post(self, path: str, body: dict) -> Any: + return self._request("POST", path, body) + + +# -- inputs from the protected checkout ------------------------------------------- + +def fetch_holds(url: str) -> Any: + if not url: + raise GateError("no hold-list URL is configured (.github/kb-gate/config.json)") + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "kb-gate"}), + timeout=20) as response: + return json.loads(response.read()) + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"hold list unreachable: {exc}") from exc + + +def verify_holds(envelope: Any, public_key, *, now: float) -> dict: + holds = verify("kb-holds", envelope, public_key) + if not 0 <= now - float(holds["issued_at"]) <= HOLDS_MAX_AGE: + raise GateError("hold list is stale (the knowledge service may be down)") + return holds + + +def parse_codeowners(text: str | None) -> list[tuple[str, tuple[str, ...]]]: + rules = [] + for line in (text or "").splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + pattern, *owners = line.split() + rules.append((pattern, tuple(owner.lstrip("@").lower() for owner in owners))) + return rules + + +def _codeowner_match(pattern: str, path: str) -> bool: + anchored = pattern.startswith("/") + pattern = pattern.lstrip("/") + if pattern.endswith("/"): + pattern += "**" + candidates = [path] if anchored or "/" in pattern.rstrip("*") else \ + [path] + [path.split("/", i)[-1] for i in range(1, path.count("/") + 1)] + for candidate in candidates: + if fnmatch.fnmatchcase(candidate, pattern) or fnmatch.fnmatchcase(candidate, pattern + "/**") \ + or (pattern.endswith("/**") and candidate.startswith(pattern[:-2])): + return True + return False + + +def code_owners(rules: list[tuple[str, tuple[str, ...]]], path: str) -> tuple[str, ...]: + """Owners of ``path``: the LAST matching line wins. Team entries never + match a person here (their membership is not readable with the job token), + so a team-only path cannot be human-approved through this verifier.""" + owners: tuple[str, ...] = () + for pattern, names in rules: + if _codeowner_match(pattern, path): + owners = names + return tuple(name for name in owners if "/" not in name) + + +# -- the verification of one change ------------------------------------------------ + +@dataclass +class Context: + git: Git + github: Any + repository: str + public_key: Any # None when .github/kb-gate.pub is missing + holds_loader: Callable[[], Any] + codeowners: list[tuple[str, tuple[str, ...]]] + now: float + _holds: dict | None = None + + def holds(self) -> dict: + if self._holds is None: + self._holds = verify_holds(self.holds_loader(), self.public_key, now=self.now) + return self._holds + + +def governed(path: str) -> bool: + """A page the knowledge lifecycle governs (what ``knowledge_files`` reads).""" + return path.startswith(GOVERNED_PREFIXES) and path.endswith(GOVERNED_SUFFIXES) + + +def touches_knowledge(entries: Iterable[Mapping[str, str]]) -> bool: + return any(entry["path"].startswith(KNOWLEDGE_PREFIX) for entry in entries) + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def verdict_envelopes(comments: Iterable[Mapping[str, Any]]) -> list[Any]: + out = [] + for comment in comments: + body = str(comment.get("body") or "") + if VERDICT_MARKER not in body: + continue + rest = body.split(VERDICT_MARKER, 1)[1] + start = rest.find("```json") + if start < 0: + continue + rest = rest[start + len("```json"):] + end = rest.find("```") + try: + out.append(json.loads(rest[:end] if end >= 0 else rest)) + except ValueError: + continue + return out + + +def find_verdict(ctx: Context, pr: int, head_sha: str) -> tuple[dict | None, list[str]]: + """The newest valid verdict for exactly this PR head, and why others failed.""" + comments = ctx.github.get_all(f"/repos/{ctx.repository}/issues/{pr}/comments") + valid, rejected = [], [] + for envelope in verdict_envelopes(comments): + try: + verdict = verify("kb-gate-verdict", envelope, ctx.public_key) + check_binding(verdict, repository=ctx.repository, pr=pr, head_sha=head_sha, now=ctx.now) + valid.append(verdict) + except (SignatureError, VerdictError, KeyError, TypeError, ValueError) as exc: + rejected.append(str(exc)) + if not valid: + return None, rejected + return max(valid, key=lambda v: float(v["issued_at"])), rejected + + +def _owner_dir(path: str) -> str: + return str(PurePosixPath(path).parent) + + +def _scope_root(path: str) -> str: + parts = PurePosixPath(path).parts + return "/".join(parts[:2]) if parts[:1] == ("repos",) else parts[0] + + +def context_paths(touched: set[str], owner_dirs: set[str], post: Mapping[str, str], + base: Mapping[str, str]) -> Callable[[str], bool]: + """The pages whose change since the judged base invalidates L2: the touched + pages, every page in their owner directories, the scope's _routes.yaml and + _index.md files (scope root and owner directories), and pages holding rules + the change supersedes.""" + roots = {_scope_root(path) for path in touched} + dirs = owner_dirs | {_owner_dir(path) for path in touched} + superseded: set[str] = set() + for path in touched: + text = post.get(path) + if not text or not path.endswith(".md"): + continue + try: + sections = Page.parse(text).rules() + except (LifecycleError, ValueError): + continue + for section in sections: + try: + target = section.footer.supersedes + except LifecycleError: + continue + for rule_id in [t.strip() for t in target.split(",") if t.strip()]: + superseded.update(p for p, t in base.items() + if p.endswith(".md") and rule_id in all_rule_ids({p: t})) + + def relevant(path: str) -> bool: + name = PurePosixPath(path).name + parent = _owner_dir(path) + return (path in touched or path in superseded or parent in dirs + or (name in ("_routes.yaml", "_index.md") and parent in roots | dirs)) + return relevant + + +def verify_change(ctx: Context, *, pr: dict, head_sha: str, pre: str, post: str, final: str, + effective_base: str, stage: str) -> list[str]: + """Problems with one PR's knowledge change (empty when it passes). + + ``pre``/``post`` are the commits (or trees) around this PR's change as it + is applied; ``final`` is the tree that would land; ``effective_base`` is + the state the change is applied on (current main, or the first parent of + the merge-group segment).""" + number = int(pr["number"]) + entries = ctx.git.raw_diff(pre, post) + if not touches_knowledge(entries): + return [] + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + problems: list[str] = [] + labels = {str(label.get("name")) for label in pr.get("labels") or []} + if pr.get("state") != "open": + problems.append(f"PR #{number} is not open") + if str((pr.get("head") or {}).get("sha")) != head_sha: + problems.append(f"PR #{number} head moved from {head_sha[:12]}") + if stage == "merge_group" and pr.get("draft"): + problems.append(f"PR #{number} is a draft (paused)") + if HOLD_LABEL in labels: + problems.append(f"PR #{number} carries {HOLD_LABEL}") + + pre_files, post_files = ctx.git.knowledge_files(pre), ctx.git.knowledge_files(post) + final_files = ctx.git.knowledge_files(final) + changes = [Change(e["path"], e["status"] if e["status"] in "AMD" else "M", e["old_mode"], e["new_mode"]) + for e in entries] + for e in entries: + if e["status"] not in ("A", "M", "D"): + problems.append(f"{e['path']}: change type {e['status']} is not allowed") + touched = {e["path"][len(KNOWLEDGE_PREFIX):] for e in entries if e["path"].startswith(KNOWLEDGE_PREFIX)} + + try: + holds = ctx.holds() + if holds.get("global"): + problems.append("the knowledge service holds every repository (global pause)") + if number in {int(p) for p in holds.get("prs") or []}: + problems.append(f"PR #{number} is on the service's hold list") + held = set(holds.get("repos") or []) + # a scope is its repository name, or "general" for knowledge/general/ + scopes = {repo_scope(p).split("/")[-1] for p in touched if p.startswith(("repos/", "general/"))} + for scope in sorted(scopes & held): + problems.append(f"repository {scope} is paused by the knowledge service") + except (GateError, SignatureError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems.append(f"hold list: {exc}") + + verdict, rejected = find_verdict(ctx, number, head_sha) + if verdict is None: + detail = f" (rejected: {'; '.join(rejected[:3])})" if rejected else "" + problems.append(f"no valid signed verdict for PR #{number} at {head_sha[:12]}{detail}") + # still report L1, so the author sees every structural problem at once + result = check_changeset(pre_files, post_files, changes, external_texts={}) + return problems + [f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues] + + if not manifests_equal(verdict.get("manifest") or [], entries): + problems.append("the signed patch manifest does not match the change as applied") + if verdict.get("facts"): + problems.append("the verdict carries upstream fact attestations this verifier cannot re-check") + + human = verdict["source"] == "human-approved" + # a maintainer's approval replaces the auto-merge whitelist (knowledge/tools, + # skills, scripts, mixed PRs) and L2, never the rest of L1: non-governed + # paths are not pages, so the page lifecycle checks run on governed ones only + l1_changes = [c for c in changes if governed(c.path)] if human else changes + result = check_changeset(pre_files, post_files, l1_changes, + external_texts=ctx.git.external_texts(final), + release=str(verdict.get("release") or "")) + waived = WHITELIST_CODES if human else () + problems.extend(f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues if i.code not in waived) + # what this change retires or purges must not be cited anywhere in the tree + # that would land (a PR queued behind it, or main, may have added a citation) + gone = set(result.retired) | set(result.purged) + if gone: + try: + problems.extend(f"L1 {i.code} {i.path} {i.detail} (in the tree that would land)" + for i in dangling_refs(final_files, gone)) + except (LifecycleError, ValueError) as exc: + problems.append(f"L1 the tree that would land does not parse: {exc}") + + if verdict["source"] == "auto": + if result.external_refs: + problems.append("retired rules are still cited outside knowledge/ (needs a companion PR)") + signed_blocks = verdict.get("blocks") or [] + if {b.block_id for b in result.blocks} != {b.get("block_id") for b in signed_blocks}: + problems.append("the signed block table does not cover exactly the changed blocks") + if any(b.get("verdict") != "pass" for b in signed_blocks): + problems.append("a signed block did not pass") + owner_dirs = {_owner_dir(b.path) for b in result.blocks} + consistency = verdict.get("consistency") or [] + if {c.get("owner_dir") for c in consistency} != owner_dirs: + problems.append("the consistency judgement does not cover exactly the changed owner directories") + for item in consistency: + directory = item.get("owner_dir") + actual = {p: _sha256(t) for p, t in final_files.items() + if _owner_dir(p) == directory and p.endswith(".md")} + if item.get("verdict") != "consistent": + problems.append(f"{directory}: consistency was not judged consistent") + if item.get("pages") != actual: + problems.append(f"{directory}: pages changed since the consistency judgement") + base_sha = str(verdict["context_base_sha"]) + try: + if not ctx.git.is_ancestor(base_sha, effective_base): + problems.append("the judged context base is not an ancestor of the effective base") + else: + relevant = context_paths(touched, owner_dirs, post_files, pre_files) + moved = [n[len(KNOWLEDGE_PREFIX):] for n in ctx.git.changed_names(base_sha, effective_base)] + stale = sorted(p for p in moved if relevant(p)) + if stale: + problems.append("context changed since the verdict was judged: " + ", ".join(stale[:5])) + except GateError as exc: + problems.append(f"context base: {exc}") + else: + problems.extend(_human_approval_problems(ctx, pr, verdict, head_sha, labels)) + return problems + + +def _human_approval_problems(ctx: Context, pr: dict, verdict: dict, head_sha: str, + labels: set[str]) -> list[str]: + problems = [] + if HUMAN_LABEL not in labels: + problems.append(f"{HUMAN_LABEL} was removed") + number = int(pr["number"]) + reviewers = {str(r).lower() for r in verdict.get("reviewers") or []} + approvers = set() + for review_id in verdict.get("review_ids") or []: + review = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}/reviews/{int(review_id)}") + login = str((review.get("user") or {}).get("login") or "").lower() + if review.get("state") != "APPROVED": + problems.append(f"review {review_id} is no longer APPROVED") + elif review.get("commit_id") != head_sha: + problems.append(f"review {review_id} approved another head") + elif login not in reviewers: + problems.append(f"review {review_id} is by someone the verdict does not name") + else: + approvers.add(login) + # knowledge maintainers = the CODEOWNERS of the gate itself. Governed pages + # deliberately have no code owners (that would force a review on every auto + # PR); paths outside the whitelist get GitHub's native code-owner review. + maintainers = set(code_owners(ctx.codeowners, MAINTAINER_PATH)) + if not approvers & maintainers: + problems.append("no valid approving review by a knowledge maintainer (CODEOWNERS of .github/kb-gate/)") + return problems + + +def tree_problems(git: Git, base: str, final: str) -> list[str]: + """L1 on the tree that would land: no tree-level issue that base lacks.""" + before = {(i.code, i.detail) for i in check_tree(git.knowledge_files(base))} + return [f"L1 {i.code} {i.path} {i.detail}".rstrip() + for i in check_tree(git.knowledge_files(final)) if (i.code, i.detail) not in before] + + +# -- the two stages ----------------------------------------------------------------- + +def verify_pr(ctx: Context, number: int, *, base_ref: str = "main", + remote: str = "origin") -> tuple[str, list[str]]: + """(the head SHA verified, its problems). The status goes on THAT head, so + a push during verification can only leave the new head without a status.""" + pr = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}") + head = str(pr["head"]["sha"]) + return head, _verify_pr_head(ctx, pr, head, base_ref=base_ref, remote=remote) + + +def _verify_pr_head(ctx: Context, pr: dict, head: str, *, base_ref: str, remote: str) -> list[str]: + number = int(pr["number"]) + ctx.git.run("fetch", "--quiet", "--no-tags", remote, + f"+refs/pull/{number}/head:refs/kb-gate/pr-{number}", f"+refs/heads/{base_ref}:refs/kb-gate/base") + if ctx.git.rev(f"refs/kb-gate/pr-{number}") != head: + return [f"fetched head does not match PR #{number}'s head {head[:12]} (retry)"] + base_tip = ctx.git.rev("refs/kb-gate/base") + merge_base = ctx.git.merge_base(base_tip, head) + if not touches_knowledge(ctx.git.raw_diff(merge_base, head)): + return [] + try: + final = ctx.git.merge_tree(base_tip, head) + except GateError as exc: + return [str(exc)] + problems = verify_change(ctx, pr=pr, head_sha=head, pre=merge_base, post=head, final=final, + effective_base=base_tip, stage="pull_request") + return problems + tree_problems(ctx.git, base_tip, final) + + +def _pr_for_head(ctx: Context, sha: str) -> dict: + matches = [pr for pr in ctx.github.get_all(f"/repos/{ctx.repository}/pulls?state=open") + if str((pr.get("head") or {}).get("sha")) == sha] + if len(matches) != 1: + raise GateError(f"{len(matches)} open PRs have head {sha[:12]}; the group cannot be attributed") + return ctx.github.get(f"/repos/{ctx.repository}/pulls/{int(matches[0]['number'])}") + + +def verify_merge_group(ctx: Context, head_sha: str, *, base_ref: str = "main", + remote: str = "origin") -> list[str]: + """Every segment of the group, each against its own first parent.""" + ctx.git.run("fetch", "--quiet", "--no-tags", remote, f"+refs/heads/{base_ref}:refs/kb-gate/base") + main_tip = ctx.git.rev("refs/kb-gate/base") + if not ctx.git.is_ancestor(main_tip, head_sha): + return [f"the merge group {head_sha[:12]} is not built on the current {base_ref}"] + chain = ctx.git.first_parent_chain(main_tip, head_sha) + if not chain or chain[-1] != head_sha: + return ["the merge group has no segments"] + problems: list[str] = [] + any_knowledge = False + for commit in chain: + parents = ctx.git.parents(commit) + if len(parents) != 2: + return [f"segment {commit[:12]} is not a two-parent merge commit; the group cannot be attributed"] + first, second = parents + if not touches_knowledge(ctx.git.raw_diff(first, commit)): + continue + any_knowledge = True + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + try: + pr = _pr_for_head(ctx, second) + except GateError as exc: + return [str(exc)] + found = verify_change(ctx, pr=pr, head_sha=second, pre=first, post=commit, final=head_sha, + effective_base=first, stage="merge_group") + problems.extend(f"PR #{pr['number']}: {p}" for p in found) + if any_knowledge: + problems.extend(tree_problems(ctx.git, main_tip, head_sha)) + return problems + + +# -- entry point ---------------------------------------------------------------------- + +def _load_context(args, github) -> Context: + root = Path(args.root) + config_path = Path(__file__).resolve().parent / "config.json" + config = json.loads(config_path.read_text()) if config_path.is_file() else {} + pub = root / ".github" / "kb-gate.pub" + public_key = load_public_key(pub.read_text()) if pub.is_file() else None + codeowners_path = root / ".github" / "CODEOWNERS" + holds_url = str(config.get("holds_url") or "") + return Context( + git=Git(root), github=github, repository=args.repository, public_key=public_key, + holds_loader=lambda: fetch_holds(holds_url), + codeowners=parse_codeowners(codeowners_path.read_text() if codeowners_path.is_file() else ""), + now=time.time(), + ) + + +def main(argv: list[str] | None = None, *, github=None) -> int: + parser = argparse.ArgumentParser(prog="kb-gate") + sub = parser.add_subparsers(dest="stage", required=True) + pr_stage = sub.add_parser("pr") + pr_stage.add_argument("--pr", type=int, required=True) + pr_stage.add_argument("--post-status", action="store_true") + pr_stage.add_argument("--target-url", default="") + group = sub.add_parser("merge-group") + group.add_argument("--head-sha", required=True) + for stage in (pr_stage, group): + stage.add_argument("--repository", required=True) + stage.add_argument("--root", default=".") + stage.add_argument("--base-ref", default="main") + args = parser.parse_args(argv) + github = github or GitHub(args.repository, os.environ.get("GITHUB_TOKEN", ""), + os.environ.get("GITHUB_API_URL", "https://api.github.com")) + ctx = _load_context(args, github) + head = "" + try: + if args.stage == "pr": + head = str(github.get(f"/repos/{args.repository}/pulls/{args.pr}")["head"]["sha"]) + head, problems = verify_pr(ctx, args.pr, base_ref=args.base_ref) + else: + problems = verify_merge_group(ctx, args.head_sha, base_ref=args.base_ref) + except (GateError, SignatureError, VerdictError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems = [f"verifier error: {exc}"] + for problem in problems: + print(f"kb-gate: {problem}") + print("kb-gate: " + ("FAIL" if problems else "PASS")) + if args.stage == "pr" and args.post_status and head: + description = (problems[0] if problems else "knowledge verdict verified")[:139] + github.post(f"/repos/{args.repository}/statuses/{head}", { + "state": "failure" if problems else "success", "context": STATUS_CONTEXT, + "description": description, **({"target_url": args.target_url} if args.target_url else {})}) + return 0 + return 1 if problems else 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/.github/kb-gate/l1.py b/.github/kb-gate/l1.py new file mode 100644 index 00000000..fd75d6d6 --- /dev/null +++ b/.github/kb-gate/l1.py @@ -0,0 +1,503 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/l1.py -- do not edit here. +"""L1: the deterministic half of the knowledge quality gate. + +Two entry points, both pure functions over ``{knowledge-relative path: text}`` +mappings so the service, the gate and the vendored verifier agree exactly: + +* ``check_tree(files)`` — structural issues in one tree (footers, supersede + links, tombstones, ``_routes.yaml``). +* ``check_changeset(base, head, changes)`` — what one change set may do: only + whitelisted files and modes, no new duplicate or reused rule IDs, no dangling + references to rules it retired or purged, no NEW tree issues, and a + classification of every change into blocks: + + ``rule`` a rule section added / edited / retired / superseded / purged. + Needs a per-block L2 verdict (keyed by ``block_id``). + ``prose`` any other wording change (page head, non-rule sections, + guides, ``_routes.yaml``). Needs an L2 prose verdict. + ``mechanical`` ``updated:``/``sources:`` frontmatter, index lines for new + pages, tombstones. Recomputed here; a mismatch is an issue, + never a block a verdict could cover. + +Standard library + PyYAML + ``lifecycle``/``ops`` only (vendored by kb-gate). +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Iterable, Mapping + +import yaml + +try: + from .lifecycle import ( + FRONTMATTER, LifecycleError, Page, Section, expected_sources, + ) + from .ops import ( + INDEX_NAME, TOMBSTONES_NAME, all_rule_ids, all_tombstoned_ids, + load_tombstones, page_over_capacity, render_tombstones, + ) +except ImportError: # pragma: no cover - vendored verifier bundle + from lifecycle import ( # type: ignore[no-redef] + FRONTMATTER, LifecycleError, Page, Section, expected_sources, + ) + from ops import ( # type: ignore[no-redef] + INDEX_NAME, TOMBSTONES_NAME, all_rule_ids, all_tombstoned_ids, + load_tombstones, page_over_capacity, render_tombstones, + ) + +ROUTES_NAME = "_routes.yaml" +KNOWLEDGE_PREFIX = "knowledge/" +_WHITELIST = ( + re.compile(r"knowledge/(?:repos/[^/]+|general)/(?:[^/]+/)*[^/]+\.md"), + re.compile(r"knowledge/repos/[^/]+/_routes\.yaml"), + re.compile(r"knowledge/(?:repos/[^/]+|general)/_tombstones\.yaml"), +) +_DENY = re.compile(r"knowledge/(?:skills|tools|\.claude)/|knowledge/[^/]+$") +_REGULAR_MODE = "100644" +_INDEX_LINE = re.compile(r"- \[[^\]\n]+\]\((?P[^)/\s]+\.md)\)") + + +@dataclass(frozen=True) +class Issue: + code: str + path: str + detail: str + + def to_dict(self) -> dict: + return {"code": self.code, "path": self.path, "detail": self.detail} + + +@dataclass(frozen=True) +class Change: + """One changed path as git reports it (repository-relative).""" + + path: str + status: str # A, M, D (renames are reported as D + A) + old_mode: str = "" + new_mode: str = "" + + +@dataclass(frozen=True) +class Block: + kind: str # rule | prose + path: str # knowledge-relative + rule_id: str # "" for prose + op: str # add | edit | retire | supersede | purge | prose + sha256: str # of the text the verdict judges + + @property + def block_id(self) -> str: + identity = json.dumps([self.kind, self.path, self.rule_id, self.op, self.sha256], + separators=(",", ":")) + return hashlib.sha256(identity.encode("utf-8")).hexdigest() + + def to_dict(self) -> dict: + return {"block_id": self.block_id, "kind": self.kind, "path": self.path, + "rule_id": self.rule_id, "op": self.op, "sha256": self.sha256} + + +@dataclass(frozen=True) +class ChangesetResult: + issues: tuple[Issue, ...] + blocks: tuple[Block, ...] + retired: tuple[str, ...] + purged: tuple[str, ...] + external_refs: tuple[tuple[str, str], ...] = () # (rule_id, repo path) + + @property + def ok(self) -> bool: + return not self.issues + + +def _sha(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _norm(text: str) -> str: + """Section text without trailing blank lines: appending a section adds a + separator to the previous one, which is layout, not a change of words.""" + return text.rstrip("\n") + "\n" + + +# -- tree-level ---------------------------------------------------------------- + +def _rule_pages(files: Mapping[str, str]) -> dict[str, Page]: + pages: dict[str, Page] = {} + for path, text in files.items(): + if path.endswith(".md"): + page = Page.parse(text) + if page.rules(): + pages[path] = page + return pages + + +def check_tree(files: Mapping[str, str]) -> list[Issue]: + issues: list[Issue] = [] + footers: dict[str, tuple[str, object]] = {} + for path, page in _rule_pages(files).items(): + for section in page.rules(): + try: + footer = section.footer + except LifecycleError as exc: + issues.append(Issue("footer_invalid", path, f"{section.rule_id}: {exc}")) + continue + footers.setdefault(section.rule_id, (path, footer)) + for rule_id, (path, footer) in footers.items(): + if footer.superseded_by: + target = footers.get(footer.superseded_by) + if target is None or target[1].supersedes != rule_id: + issues.append(Issue("supersede_link_broken", path, + f"{rule_id} -> {footer.superseded_by} has no matching supersedes")) + if footer.supersedes: + source = footers.get(footer.supersedes) + if source is not None and source[1].superseded_by != rule_id: + issues.append(Issue("supersede_link_broken", path, + f"{rule_id} supersedes {footer.supersedes}, which does not name it")) + try: + tombstoned = all_tombstoned_ids(files) + except (LifecycleError, yaml.YAMLError) as exc: + issues.append(Issue("tombstones_invalid", "", str(exc))) + tombstoned = set() + for rule_id in sorted(tombstoned & set(all_rule_ids(files))): + issues.append(Issue("tombstoned_id_reused", "", rule_id)) + for path, text in files.items(): + if PurePosixPath(path).name == ROUTES_NAME: + issues.extend(_check_routes(path, text, files)) + return issues + + +def _check_routes(path: str, text: str, files: Mapping[str, str]) -> list[Issue]: + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + return [Issue("routes_invalid", path, f"not YAML: {exc}")] + if not isinstance(data, dict) or data.get("schema_version") != 1: + return [Issue("routes_invalid", path, "schema_version must be 1")] + issues = [] + seen: set[str] = set() + for index, owner in enumerate(data.get("owners") or []): + if not isinstance(owner, dict) or not owner.get("owner") or not owner.get("path"): + issues.append(Issue("routes_invalid", path, f"owners[{index}] needs owner and path")) + continue + if owner["owner"] in seen: + issues.append(Issue("routes_invalid", path, f"duplicate owner {owner['owner']}")) + seen.add(owner["owner"]) + if str(owner["path"]) not in files: + issues.append(Issue("routes_page_missing", path, str(owner["path"]))) + for key in ("signals", "scope_prefixes"): + if not isinstance(owner.get(key) or [], list): + issues.append(Issue("routes_invalid", path, f"{owner['owner']}.{key} must be a list")) + models = data.get("models") + if models is not None and (not isinstance(models, dict) or not models.get("dir") or not models.get("page")): + issues.append(Issue("routes_invalid", path, "models needs dir and page")) + return issues + + +# -- change-set level ---------------------------------------------------------- + +def _whitelist_issues(changes: Iterable[Change]) -> list[Issue]: + issues = [] + for change in changes: + path = change.path + if not path.startswith(KNOWLEDGE_PREFIX) or _DENY.match(path) \ + or not any(p.fullmatch(path) for p in _WHITELIST): + issues.append(Issue("path_not_whitelisted", path, "only knowledge rule/guide pages, _routes.yaml and _tombstones.yaml auto-merge")) + continue + if change.status != "D" and change.new_mode != _REGULAR_MODE: + issues.append(Issue("mode_not_regular", path, f"mode {change.new_mode or '?'} (need 100644)")) + return issues + + +def _text_issues(path: str, text: str | None) -> list[Issue]: + if text is None: + return [] + if "\x00" in text or "�" in text: + return [Issue("not_utf8_text", path, "binary or non-UTF-8 content")] + return [] + + +def _frontmatter_mechanical(path: str, base: Page, head: Page) -> list[Issue]: + """Only ``updated:`` and ``sources:`` may change, and sources exactly as recomputed.""" + base_data, head_data = base.frontmatter_data(), head.frontmatter_data() + issues = [] + for key in sorted(set(base_data) | set(head_data)): + if key in ("updated", "sources"): + continue + if base_data.get(key) != head_data.get(key): + issues.append(Issue("frontmatter_changed", path, f"{key} may not change")) + if str(head_data.get("updated", "")) < str(base_data.get("updated", "")): + issues.append(Issue("frontmatter_changed", path, "updated went backwards")) + want = expected_sources(base.sources(), head) + if head.sources() != want: + issues.append(Issue("sources_mismatch", path, f"expected {want}, got {head.sources()}")) + return issues + + +def _new_page_mechanical(path: str, head: Page) -> list[Issue]: + data = head.frontmatter_data() + issues = [] + if data.get("type") != "rule": + issues.append(Issue("new_page_invalid", path, "a new page must be type: rule")) + if not str(data.get("title") or "").strip(): + issues.append(Issue("new_page_invalid", path, "a new page needs a title")) + want = expected_sources([], head) + if head.sources() != want: + issues.append(Issue("sources_mismatch", path, f"expected {want}, got {head.sources()}")) + return issues + + +def _classify_page(path: str, base_text: str | None, head_text: str | None, + blocks: list[Block], issues: list[Issue], + retired: list[str], purged: list[tuple[str, str]], + purge_retired_at: dict[str, str]) -> None: + base = Page.parse(base_text) if base_text is not None else None + head = Page.parse(head_text) if head_text is not None else None + if head is None: + issues.append(Issue("page_deleted", path, "pages are never deleted; purge rules instead")) + return + base_is_rule = base is not None and ( + base.frontmatter_data().get("type") == "rule" or bool(base.rules())) + is_rule_page = base_is_rule or head.frontmatter_data().get("type") == "rule" + if base_is_rule and head.frontmatter_data().get("type") != base.frontmatter_data().get("type"): + # a page may not leave the lifecycle by changing its type + issues.append(Issue("page_type_changed", path, "a rule page keeps its type")) + if not is_rule_page: + if base is None or base.render() != head.render(): + blocks.append(Block("prose", path, "", "prose", _sha(head.render()))) + return + if base is None: + issues.extend(_new_page_mechanical(path, head)) + base_rules: dict[str, Section] = {} + base_other: list[str] = [] + base_head = "" + else: + issues.extend(_frontmatter_mechanical(path, base, head)) + base_rules = {s.rule_id: s for s in base.rules()} + base_other = [_norm(s.text) for s in base.sections if not s.is_rule] + base_head = base.head + head_rules = {s.rule_id: s for s in head.rules()} + head_other = [_norm(s.text) for s in head.sections if not s.is_rule] + if (base is None and head.head.strip() != f"# {head.frontmatter_data().get('title', '')}") \ + or (base is not None and head.head != base_head) or head_other != base_other: + blocks.append(Block("prose", path, "", "prose", + _sha(head.head + "".join(head_other)))) + for rule_id, section in head_rules.items(): + old = base_rules.get(rule_id) + try: + footer = section.footer + old_footer = old.footer if old is not None else None + except LifecycleError as exc: + issues.append(Issue("footer_invalid", path, f"{rule_id}: {exc}")) + continue + if old is None: + if footer.status != "active": + issues.append(Issue("rule_added_retired", path, rule_id)) + blocks.append(Block("rule", path, rule_id, "add", _sha(_norm(section.text)))) + continue + if _norm(old.text) == _norm(section.text): + continue + if old_footer.status == "retired": + issues.append(Issue("retired_rule_changed", path, f"{rule_id} is retired and frozen")) + continue + if old_footer.protected and not footer.protected: + issues.append(Issue("protection_removed", path, rule_id)) + if footer.status == "retired": + if section.body_without_footer != old.body_without_footer: + issues.append(Issue("retire_changed_text", path, f"{rule_id}: retiring may not reword")) + if old_footer.protected or footer.protected: + issues.append(Issue("protected_rule_retired", path, rule_id)) + op = "supersede" if footer.reason == "superseded" else "retire" + retired.extend(dict.fromkeys((rule_id, *section.nested_rule_ids))) + blocks.append(Block("rule", path, rule_id, op, _sha(_norm(section.text)))) + else: + if footer != old_footer: + issues.append(Issue("footer_changed", path, f"{rule_id}: an edit keeps its footer")) + if old_footer.protected or footer.protected: + issues.append(Issue("protected_rule_edited", path, rule_id)) + if set(section.nested_rule_ids) != set(old.nested_rule_ids): + # a nested rule leaves only by retiring and purging its parent + issues.append(Issue("nested_rule_ids_changed", path, + f"{rule_id}: {sorted(set(old.nested_rule_ids) ^ set(section.nested_rule_ids))}")) + blocks.append(Block("rule", path, rule_id, "edit", _sha(_norm(section.text)))) + for rule_id, old in base_rules.items(): + if rule_id in head_rules: + continue + if old.footer.status != "retired": + issues.append(Issue("active_rule_removed", path, f"{rule_id}: retire before purging")) + continue + if old.footer.protected: + issues.append(Issue("protected_rule_purged", path, rule_id)) + purged.extend((rid, path) for rid in dict.fromkeys((rule_id, *old.nested_rule_ids))) + purge_retired_at[rule_id] = old.footer.retired_at + blocks.append(Block("rule", path, rule_id, "purge", _sha(_norm(old.text)))) + + +def _index_mechanical(path: str, base_text: str | None, head_text: str | None, + new_pages: set[str], issues: list[Issue], blocks: list[Block], + head: Mapping[str, str] | None = None) -> None: + if base_text is None or head_text is None: + issues.append(Issue("index_added_or_deleted", path, "index files are never created or deleted by a change set")) + return + directory = str(PurePosixPath(path).parent) + expected_new = sorted(PurePosixPath(p).name for p in new_pages + if str(PurePosixPath(p).parent) == directory + and p.endswith(".md") and PurePosixPath(p).name != INDEX_NAME) + # pages already in the directory but missing from the old index may be + # linked too (the release sweep's T1 repair); new pages MUST be + allowed = set(expected_new) | { + PurePosixPath(p).name for p in (head or {}) + if str(PurePosixPath(p).parent) == directory and p.endswith(".md") + and PurePosixPath(p).name != INDEX_NAME and f"]({PurePosixPath(p).name})" not in base_text + } + for name in expected_new: # required whatever else the index change does + if f"]({name})" not in head_text: + issues.append(Issue("index_missing", path, f"new page {name} is not linked")) + if not head_text.startswith(base_text.rstrip("\n") + "\n") and head_text != base_text: + blocks.append(Block("prose", path, "", "prose", _sha(head_text))) + return + added = head_text[len(base_text.rstrip("\n") + "\n"):].splitlines() + files = [] + for line in added: + match = _INDEX_LINE.fullmatch(line.strip()) + if match is None: + blocks.append(Block("prose", path, "", "prose", _sha(head_text))) + return + files.append(match.group("file")) + if not set(expected_new) <= set(files) or not set(files) <= allowed or len(files) != len(set(files)): + issues.append(Issue("index_mismatch", path, + f"index lines must cover new pages {expected_new} and only link unlisted pages; got {files}")) + + +def check_changeset( + base: Mapping[str, str], + head: Mapping[str, str], + changes: Iterable[Change], + *, + external_texts: Mapping[str, str] | None = None, + release: str = "", +) -> ChangesetResult: + """Gate one change set. ``base``/``head`` map knowledge-relative paths + (``repos/...``) to text; ``changes`` are repository-relative git changes. + ``external_texts`` (repo path -> text, outside ``knowledge/``) is scanned + for references to rules this change retires or purges. ``release`` is the + trusted current release (from the signed verdict); when given, purges must + carry it as ``purged_at``. A purge is always rejected in the release that + retired the rule.""" + changes = list(changes) + issues = _whitelist_issues(changes) + blocks: list[Block] = [] + retired: list[str] = [] + purged: list[tuple[str, str]] = [] # (rule id, page) + purge_retired_at: dict[str, str] = {} + knowledge_paths = [c.path[len(KNOWLEDGE_PREFIX):] for c in changes + if c.path.startswith(KNOWLEDGE_PREFIX)] + for path in knowledge_paths: + issues.extend(_text_issues(path, head.get(path))) + if path.endswith(".md") and head.get(path) is not None: + over = page_over_capacity(head[path]) + if over: + issues.append(Issue("page_over_capacity", path, over)) + new_pages = {p for p in knowledge_paths if p not in base and p in head} + for page_path in sorted(new_pages): + index = str(PurePosixPath(page_path).with_name(INDEX_NAME)) + if page_path.endswith(".md") and PurePosixPath(page_path).name != INDEX_NAME \ + and index not in knowledge_paths: + issues.append(Issue("index_missing", page_path, f"{index} must list the new page")) + tombstone_paths = [] + for path in sorted(knowledge_paths): + name = PurePosixPath(path).name + if name == INDEX_NAME: + _index_mechanical(path, base.get(path), head.get(path), new_pages, issues, blocks, head) + elif name == TOMBSTONES_NAME: + tombstone_paths.append(path) + elif name == ROUTES_NAME: + blocks.append(Block("prose", path, "", "prose", _sha(head.get(path) or ""))) + elif path.endswith(".md"): + try: + _classify_page(path, base.get(path), head.get(path), blocks, issues, + retired, purged, purge_retired_at) + except (LifecycleError, yaml.YAMLError) as exc: + issues.append(Issue("page_unparseable", path, str(exc))) + # tombstones: exactly the IDs purged in this change set, appended + for path in tombstone_paths: + try: + before = load_tombstones(base.get(path)) + after = load_tombstones(head.get(path)) + except (LifecycleError, yaml.YAMLError) as exc: + issues.append(Issue("tombstones_invalid", path, str(exc))) + continue + scope = str(PurePosixPath(path).parent) + purged_here = [rid for rid, page_path in purged if page_path.startswith(scope + "/")] + appended = after[len(before):] + if after[:len(before)] != before or sorted(i["id"] for i in appended) != sorted(purged_here): + issues.append(Issue("tombstones_mismatch", path, f"expected appended tombstones for {sorted(purged_here)}")) + for item in appended: + retired_at = purge_retired_at.get(str(item["id"])) + purged_at = str(item.get("purged_at") or "") + if retired_at is None: + continue # a nested ID; its parent carries the timing + if not purged_at or purged_at == retired_at: + issues.append(Issue("purge_too_early", path, + f"{item['id']}: purged in the release it was retired ({retired_at})")) + if release and purged_at != release: + issues.append(Issue("purge_release_mismatch", path, + f"{item['id']}: purged_at {purged_at} is not this release {release}")) + if head.get(path) != render_tombstones(after): + issues.append(Issue("tombstones_mismatch", path, "tombstones are not in canonical form")) + for rule_id, _page in purged: + if rule_id not in all_tombstoned_ids(head): + issues.append(Issue("purge_without_tombstone", "", rule_id)) + # no NEW duplicate IDs, no reuse of retired or tombstoned IDs + base_ids, head_ids = all_rule_ids(base), all_rule_ids(head) + for rule_id, count in head_ids.items(): + if count > base_ids.get(rule_id, 0) and count > 1: + issues.append(Issue("duplicate_rule_id", "", rule_id)) + base_tomb = all_tombstoned_ids(base) + for block in blocks: + if block.op == "add" and (block.rule_id in base_tomb or block.rule_id in base_ids): + issues.append(Issue("rule_id_reused", block.path, block.rule_id)) + # dangling references to what this change set retired or purged + gone = set(retired) | {rid for rid, _page in purged} + if gone: + issues.extend(_dangling_refs(head, gone)) + external = tuple( + (rule_id, path) + for path, text in sorted((external_texts or {}).items()) + for rule_id in sorted(gone) if re.search(rf"(? list[Issue]: + """References in ACTIVE text to rules that are now retired or purged. + Allowed: the rule's own section, lifecycle footers, tombstones.""" + issues = [] + for path, text in head.items(): + if not path.endswith(".md"): + continue + page = Page.parse(text) + chunks = [page.head] + [ + s.body_without_footer for s in page.sections + if s.rule_id not in gone and (not s.is_rule or _active(s)) + ] + body = "\n".join(chunks) + for rule_id in sorted(gone): + if re.search(rf"(? bool: + try: + return section.footer.status == "active" + except LifecycleError: + return True diff --git a/.github/kb-gate/lifecycle.py b/.github/kb-gate/lifecycle.py new file mode 100644 index 00000000..dafe6531 --- /dev/null +++ b/.github/kb-gate/lifecycle.py @@ -0,0 +1,424 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/lifecycle.py -- do not edit here. +"""Rule pages as data: sections, lifecycle footers, and byte-preserving render. + +A rule page is frontmatter, a head (title + preamble), then sections. A section +starts at every ``## `` heading, and also at a ``### — `` heading +whose enclosing ``## `` section is not itself a rule (owner pages group rules +under topic headings). A section whose heading is ``## <ID> — <title>`` or +``### <ID> — <title>`` is a rule; ``###`` rules INSIDE a ``##`` rule stay part +of it as nested rules. Each rule may end +with one machine-readable footer, an HTML comment that renders invisibly:: + + <!-- kb:rule status=retired since=v0.30.0 retired_at=v0.31.0 reason=upstream-removed evidence="PR #8201" superseded_by=BENCH-1c --> + +No footer means ``status=active``, which is every rule written before the +lifecycle existed. Parsing then rendering an untouched page returns its exact +bytes, so an operation changes only the sections it targets. + +This module needs only the standard library and PyYAML and imports nothing +else from the package: the ``kb-gate`` verifier bundle vendors it verbatim. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field, replace +from typing import Iterable + +import yaml + +RULE_HEADING = re.compile(r"^(?P<level>#{2,3})\s+(?P<rule>[A-Za-z0-9][A-Za-z0-9-]{1,40})\s+[—-]\s+\S") +_TOP_RULE = re.compile(r"^##\s+[A-Za-z0-9][A-Za-z0-9-]{1,40}\s+[—-]\s+\S") +ANY_RULE_HEADING = re.compile( + r"^#{2,6}\s+(?P<rule>[A-Za-z0-9][A-Za-z0-9-]{1,40})(?=\s|$)", re.MULTILINE) +FOOTER = re.compile(r"^<!--\s*kb:rule\s+(?P<attrs>.*?)\s*-->\s*$") +_ATTR = re.compile(r'(?P<key>[a-z_]+)=(?:"(?P<quoted>[^"]*)"|(?P<bare>[^\s"]+))') +FRONTMATTER = re.compile(r"\A---\r?\n(?P<body>.*?)\r?\n---(?:\r?\n|\Z)", re.DOTALL) +CITATION = re.compile(r"\^\[(?P<ref>[^\]\n]{1,60})\]") + +STATUSES = ("active", "retired") +FOOTER_KEYS = ( + "status", "since", "retired_at", "reason", "evidence", + "supersedes", "superseded_by", "protected", +) +RETIRE_REASONS = ("upstream-removed", "superseded", "incorrect", "duplicate") + + +class LifecycleError(ValueError): + """A page, footer, or operation violates the rule lifecycle contract.""" + + +@dataclass(frozen=True) +class Footer: + status: str = "active" + since: str = "" + retired_at: str = "" + reason: str = "" + evidence: str = "" + supersedes: str = "" + superseded_by: str = "" + protected: bool = False + + @classmethod + def parse(cls, attrs: str) -> "Footer": + values: dict[str, str] = {} + consumed = 0 + for match in _ATTR.finditer(attrs): + if attrs[consumed:match.start()].strip(): + raise LifecycleError(f"malformed kb:rule footer: {attrs!r}") + key = match.group("key") + if key not in FOOTER_KEYS: + raise LifecycleError(f"unknown kb:rule footer key: {key}") + if key in values: + raise LifecycleError(f"repeated kb:rule footer key: {key}") + values[key] = match.group("quoted") if match.group("quoted") is not None \ + else match.group("bare") + consumed = match.end() + if attrs[consumed:].strip(): + raise LifecycleError(f"malformed kb:rule footer: {attrs!r}") + status = values.get("status", "active") + if status not in STATUSES: + raise LifecycleError(f"kb:rule status must be one of {STATUSES}: {status}") + protected = values.get("protected", "false") + if protected not in {"true", "false"}: + raise LifecycleError("kb:rule protected must be true or false") + footer = cls( + status=status, + since=values.get("since", ""), + retired_at=values.get("retired_at", ""), + reason=values.get("reason", ""), + evidence=values.get("evidence", ""), + supersedes=values.get("supersedes", ""), + superseded_by=values.get("superseded_by", ""), + protected=protected == "true", + ) + footer.check() + return footer + + def check(self) -> None: + if self.status == "retired": + missing = [key for key in ("retired_at", "reason", "evidence") + if not getattr(self, key)] + if missing: + raise LifecycleError(f"retired rule footer lacks {', '.join(missing)}") + if self.reason not in RETIRE_REASONS: + raise LifecycleError( + f"retire reason must be one of {RETIRE_REASONS}: {self.reason}") + if self.reason == "superseded" and not self.superseded_by: + raise LifecycleError("a superseded rule must name superseded_by") + elif self.retired_at or self.superseded_by: + raise LifecycleError("an active rule cannot carry retired_at/superseded_by") + + def render(self) -> str: + parts = [] + for key in FOOTER_KEYS: + value = getattr(self, key) + if key == "protected": + if value: + parts.append("protected=true") + continue + if key == "status": + parts.append(f"status={value}") + continue + if not value: + continue + if re.search(r'[\s"]', value): + if '"' in value: + raise LifecycleError(f"footer value may not contain a quote: {key}") + parts.append(f'{key}="{value}"') + else: + parts.append(f"{key}={value}") + return f"<!-- kb:rule {' '.join(parts)} -->" + + +@dataclass(frozen=True) +class Section: + """One level-two section, kept as its exact source text.""" + + text: str # includes the heading line and its trailing newline(s) + + @property + def heading(self) -> str: + return self.text.split("\n", 1)[0] + + @property + def rule_id(self) -> str: + match = RULE_HEADING.match(self.heading) + return match.group("rule") if match else "" + + @property + def level(self) -> int: + match = RULE_HEADING.match(self.heading) + return len(match.group("level")) if match else 2 + + @property + def is_rule(self) -> bool: + return bool(self.rule_id) + + def _footer_line(self) -> tuple[int, str] | None: + lines = self.text.rstrip("\n").split("\n") + for index in range(len(lines) - 1, 0, -1): + line = lines[index].strip() + if not line: + continue + return (index, line) if FOOTER.match(line) else None + return None + + @property + def footer(self) -> Footer: + found = self._footer_line() + if found is None: + return Footer() + return Footer.parse(FOOTER.match(found[1]).group("attrs")) + + @property + def has_footer(self) -> bool: + return self._footer_line() is not None + + @property + def body_without_footer(self) -> str: + """The rule's words: the section minus its footer, trailing blanks trimmed.""" + found = self._footer_line() + lines = self.text.rstrip("\n").split("\n") + if found is not None: + lines = lines[:found[0]] + return "\n".join(lines).rstrip() + "\n" + + @property + def content_sha256(self) -> str: + return hashlib.sha256(self.body_without_footer.encode("utf-8")).hexdigest() + + @property + def nested_rule_ids(self) -> tuple[str, ...]: + return tuple(m.group("rule") for m in ANY_RULE_HEADING.finditer(self.text)) + + @property + def citations(self) -> tuple[str, ...]: + return tuple(dict.fromkeys(m.group("ref").strip() for m in CITATION.finditer(self.text))) + + def with_footer(self, footer: Footer) -> "Section": + footer.check() + trailing = self.text[len(self.text.rstrip("\n")):] or "\n" + return Section(self.body_without_footer + "\n" + footer.render() + trailing) + + +@dataclass(frozen=True) +class Page: + frontmatter: str # the exact "---...---\n" block, or "" when absent + head: str # text between frontmatter and the first "## " section + sections: tuple[Section, ...] = field(default_factory=tuple) + + @classmethod + def parse(cls, text: str) -> "Page": + match = FRONTMATTER.match(text) + frontmatter = match.group(0) if match else "" + rest = text[len(frontmatter):] + chunks = _split_sections(rest) + head = chunks[0] + sections = tuple(Section(chunk) for chunk in chunks[1:]) + return cls(frontmatter, head, sections) + + def render(self) -> str: + return self.frontmatter + self.head + "".join(s.text for s in self.sections) + + # -- rules --------------------------------------------------------------- + + def rules(self) -> tuple[Section, ...]: + return tuple(s for s in self.sections if s.is_rule) + + def rule(self, rule_id: str) -> Section: + for section in self.sections: + if section.rule_id == rule_id: + return section + raise LifecycleError(f"rule {rule_id} is not on this page") + + def has_rule(self, rule_id: str) -> bool: + return any(section.rule_id == rule_id for section in self.sections) + + def replace_section(self, rule_id: str, new: Section | None) -> "Page": + out = [] + found = False + for section in self.sections: + if section.rule_id == rule_id: + found = True + if new is not None: + out.append(new) + else: + out.append(section) + if not found: + raise LifecycleError(f"rule {rule_id} is not on this page") + return replace(self, sections=tuple(out)) + + def append_section(self, section: Section) -> "Page": + sections = list(self.sections) + if sections and not sections[-1].text.endswith("\n\n"): + last = sections[-1].text + sections[-1] = Section(last.rstrip("\n") + "\n\n") + elif not sections and self.head and not self.head.endswith("\n\n"): + return replace(self, head=self.head.rstrip("\n") + "\n\n", sections=(section,)) + sections.append(section) + return replace(self, sections=tuple(sections)) + + # -- frontmatter --------------------------------------------------------- + + def frontmatter_field(self, key: str) -> str | None: + match = re.search(rf"(?m)^{re.escape(key)}:[ \t]*(?P<value>.*)$", self.frontmatter) + return match.group("value").strip() if match else None + + def with_frontmatter_field(self, key: str, value: str) -> "Page": + if not self.frontmatter: + raise LifecycleError("page has no frontmatter") + pattern = re.compile(rf"(?m)^{re.escape(key)}:[^\r\n]*$") + if not pattern.search(self.frontmatter): + raise LifecycleError(f"frontmatter has no {key} field") + return replace(self, frontmatter=pattern.sub(f"{key}: {value}", self.frontmatter, count=1)) + + def frontmatter_data(self) -> dict: + match = FRONTMATTER.match(self.frontmatter) + if not match: + return {} + try: + data = yaml.safe_load(match.group("body")) or {} + except yaml.YAMLError as exc: + raise LifecycleError(f"frontmatter is not valid YAML: {exc}") from exc + if not isinstance(data, dict): + raise LifecycleError("frontmatter must be a mapping") + return data + + def sources(self) -> list[str]: + value = self.frontmatter_data().get("sources") + if value is None: + return [] + if not isinstance(value, list): + raise LifecycleError("frontmatter sources must be a list") + return [str(item) for item in value] + + def with_sources(self, items: list[str]) -> "Page": + """Rewrite ``sources:`` in the page's own style (flow or block list).""" + match = re.search( + r"(?m)^sources:[ \t]*(?P<flow>\[.*\])?[ \t]*\r?\n(?P<block>(?:[ \t]+-[^\r\n]*\r?\n)*)", + self.frontmatter, + ) + if match is None: + raise LifecycleError("frontmatter has no sources field") + if match.group("flow") is not None or not match.group("block"): + rendered = f"sources: {render_flow_list(items)}\n" + else: + indent = re.match(r"[ \t]+", match.group("block")).group(0) + rendered = "sources:\n" + "".join( + f'{indent}- "{_quote_free(item)}"\n' for item in items) + return replace( + self, + frontmatter=self.frontmatter[:match.start()] + rendered + + self.frontmatter[match.end():], + ) + + +_FENCE = re.compile(r"^\s{0,3}(`{3,}|~{3,})") + + +def _split_sections(text: str) -> list[str]: + """Split at ``## `` headings, and at every ``### `` heading whose enclosing + ``## `` section is not a rule (so a level-three rule ends at its next + sibling); ``###`` inside a ``##`` rule stays nested. Fences never split.""" + chunks: list[str] = [] + current: list[str] = [] + fence: str | None = None + top_is_rule = False + for line in text.splitlines(keepends=True): + match = _FENCE.match(line) + if match: + marker = match.group(1) + if fence is None: + fence = marker[0] * len(marker) + elif marker[0] == fence[0] and len(marker) >= len(fence): + fence = None + split = False + if fence is None and not match: + if line.startswith("## "): + split = True + top_is_rule = bool(_TOP_RULE.match(line)) + elif line.startswith("### ") and not top_is_rule: + # every ### under a topic heading is its own section, so a + # level-three rule ends at its next sibling heading + split = True + if split: + chunks.append("".join(current)) + current = [line] + else: + current.append(line) + chunks.append("".join(current)) + return chunks + + +def parse_flow_list(raw: str) -> list[str]: + """Parse the YAML flow sequence the tree uses for ``sources:``/``tags:``.""" + raw = raw.strip() + if not (raw.startswith("[") and raw.endswith("]")): + raise LifecycleError(f"expected a [flow, list]: {raw!r}") + inner = raw[1:-1].strip() + if not inner: + return [] + items: list[str] = [] + for match in re.finditer(r'\s*(?:"(?P<q>[^"]*)"|(?P<b>[^,]+?))\s*(?:,|$)', inner): + value = match.group("q") if match.group("q") is not None else match.group("b") + if value is not None and value.strip(): + items.append(value.strip()) + return items + + +def _quote_free(item: str) -> str: + if '"' in item: + raise LifecycleError(f"list item may not contain a quote: {item!r}") + return item + + +def render_flow_list(items: Iterable[str]) -> str: + rendered = [] + for item in items: + if re.fullmatch(r"[A-Za-z0-9_./:@+-]+", item): + rendered.append(item) + else: + if '"' in item: + raise LifecycleError(f"list item may not contain a quote: {item!r}") + rendered.append(f'"{item}"') + return "[" + ", ".join(rendered) + "]" + + +def expected_sources(before: list[str], page: Page) -> list[str]: + """Deterministic ``sources:`` after a change: the old list, in order, plus + every citation an ACTIVE rule on the page makes that is not yet listed, in + order of first appearance. Nothing is ever removed (history stays cited).""" + out = list(before) + for section in page.rules(): + if section.footer.status != "active": + continue + for ref in section.citations: + if ref not in out: + out.append(ref) + return out + + +def visible_text(text: str) -> str: + """The page as reviewers and agents should see it: retired rules removed. + + A retired rule keeps its text on disk for one release cycle so the + retirement can be reviewed and reverted, but it must stop being SERVED at + once. Pages without retired rules are returned unchanged, byte for byte. + """ + if "kb:rule" not in text: + return text + page = Page.parse(text) + kept = [] + for section in page.sections: + try: + retired = section.is_rule and section.footer.status == "retired" + except LifecycleError: + retired = False # L1 rejects malformed footers; never hide on doubt + if not retired: + kept.append(section) + if len(kept) == len(page.sections): + return text + return replace(page, sections=tuple(kept)).render() + diff --git a/.github/kb-gate/ops.py b/.github/kb-gate/ops.py new file mode 100644 index 00000000..7e18421d --- /dev/null +++ b/.github/kb-gate/ops.py @@ -0,0 +1,376 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/ops.py -- do not edit here. +"""Knowledge Ops API 2.0: typed changes to rule pages, with the lifecycle. + +The v1 curation path (``KnowledgeCurator``) can only append a section. The +knowledge service needs every change a maintainer makes by hand, expressed as a +typed operation the quality gate can judge one by one: + +==================== ========================================================= +``add`` append a new rule (optionally on a new ``rules-*.md`` page) +``edit_same_meaning`` reword an active rule under the same ID (gate must confirm + the meaning is unchanged) +``replace`` retire an active rule as ``superseded`` and add its + successor under a NEW ID, atomically +``retire`` retire an active rule (upstream removed it, it was wrong, + or it duplicates another); its text stays for one cycle +``purge`` delete a retired rule's text and tombstone its ID forever +==================== ========================================================= + +``apply_operations`` is a pure function over a mapping of knowledge-relative +paths to text, so the service, the gate and the verifier all compute exactly +the same result. After the operations it writes the MECHANICAL consequences the +verifier later recomputes: ``updated:``, ``sources:`` (append-only), an index +line for every new page, and tombstones for purged IDs. + +Standard library + PyYAML + ``lifecycle`` only: vendored by the verifier bundle. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import PurePosixPath +from typing import Iterable, Mapping + +import yaml + +try: # package import; the verifier bundle vendors both modules side by side + from .lifecycle import ( + ANY_RULE_HEADING, RETIRE_REASONS, RULE_HEADING, Footer, LifecycleError, + Page, Section, expected_sources, + ) +except ImportError: # pragma: no cover - exercised by the vendored bundle + from lifecycle import ( # type: ignore[no-redef] + ANY_RULE_HEADING, RETIRE_REASONS, RULE_HEADING, Footer, LifecycleError, + Page, Section, expected_sources, + ) + +KNOWLEDGE_OPS_API_VERSION = "2.0.0" +OP_KINDS = ("add", "edit_same_meaning", "replace", "retire", "purge") +TOMBSTONES_NAME = "_tombstones.yaml" +INDEX_NAME = "_index.md" +_RULE_PAGE = re.compile(r"rules(?:-[a-z0-9][a-z0-9-]*)?\.md") +_DATE = re.compile(r"\d{4}-\d{2}-\d{2}") +_MAX_SECTION_CHARS = 16 * 1024 +# Mirrors knowledge/tools/check_knowledge_tree.py's split gate (>= 32 KiB or +# >= 500 non-empty lines fails the whole tree). +PAGE_MAX_BYTES = 32 * 1024 +PAGE_MAX_LINES = 500 + + +def page_over_capacity(text: str) -> str: + size = len(text.encode("utf-8")) + lines = sum(1 for line in text.splitlines() if line.strip()) + if size >= PAGE_MAX_BYTES or lines >= PAGE_MAX_LINES: + return f"{size} bytes / {lines} non-empty lines (limit < {PAGE_MAX_BYTES} / < {PAGE_MAX_LINES})" + return "" + + +@dataclass(frozen=True) +class KnowledgeOperation: + kind: str + page: str + rule_id: str + section_markdown: str = "" + new_rule_id: str = "" + new_page: str = "" + reason: str = "" + evidence: str = "" + page_title: str = "" + allow_protected: bool = False + + def to_dict(self) -> dict: + return {key: value for key, value in self.__dict__.items() if value not in ("", False)} + + @classmethod + def from_dict(cls, data: Mapping) -> "KnowledgeOperation": + unknown = set(data) - set(cls.__dataclass_fields__) + if unknown: + raise LifecycleError(f"unknown operation fields: {sorted(unknown)}") + return cls(**{key: data[key] for key in data}) + + +@dataclass(frozen=True) +class OperationsResult: + files: dict[str, str] # every created or changed path -> new text + touched_rules: tuple[str, ...] # rule IDs whose sections changed + created_pages: tuple[str, ...] = field(default_factory=tuple) + + +def repo_scope(page: str) -> str: + """``repos/<repo>`` or ``general`` for a knowledge-relative page path.""" + parts = PurePosixPath(page).parts + if len(parts) >= 2 and parts[0] == "repos": + return f"repos/{parts[1]}" + if parts and parts[0] == "general": + return "general" + raise LifecycleError(f"page is outside repos/<repo>/ and general/: {page}") + + +def tombstones_path(page: str) -> str: + return f"{repo_scope(page)}/{TOMBSTONES_NAME}" + + +def load_tombstones(text: str | None) -> list[dict]: + if not text: + return [] + data = yaml.safe_load(text) or {} + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise LifecycleError("tombstones must be schema_version 1") + items = data.get("ids") or [] + if not isinstance(items, list) or any( + not isinstance(item, dict) or not item.get("id") for item in items + ): + raise LifecycleError("tombstones ids must be a list of {id, ...}") + return items + + +def render_tombstones(items: list[dict]) -> str: + lines = [ + "# IDs of purged knowledge rules. An ID listed here is never reused.", + "# Written by the knowledge service; recomputed by the kb-gate verifier.", + "schema_version: 1", + "ids:", + ] + for item in items: + lines.append( + f" - {{id: {item['id']}, purged_at: {item.get('purged_at', '')}, " + f"page: {item.get('page', '')}}}" + ) + return "\n".join(lines) + "\n" + + +def all_rule_ids(files: Mapping[str, str]) -> dict[str, int]: + """Heading IDs at any level across every rule page, with their counts.""" + counts: dict[str, int] = {} + for path, text in files.items(): + if not path.endswith(".md"): + continue + for match in ANY_RULE_HEADING.finditer(_without_fences(text)): + counts[match.group("rule")] = counts.get(match.group("rule"), 0) + 1 + return counts + + +def all_tombstoned_ids(files: Mapping[str, str]) -> set[str]: + out: set[str] = set() + for path, text in files.items(): + if PurePosixPath(path).name == TOMBSTONES_NAME: + out.update(str(item["id"]) for item in load_tombstones(text)) + return out + + +def _without_fences(text: str) -> str: + out, fence = [], None + for line in text.splitlines(keepends=True): + stripped = line.lstrip() + if stripped.startswith(("```", "~~~")): + marker = stripped[:3] + fence = None if fence == marker else (fence or marker) + out.append("\n") + continue + out.append("\n" if fence else line) + return "".join(out) + + +def _check_section(section_markdown: str, rule_id: str, level: int = 2) -> Section: + text = section_markdown.strip("\n") + "\n" + section = Section(text) + marks = "#" * level + if section.rule_id != rule_id or section.level != level: + raise LifecycleError(f"section heading must be '{marks} {rule_id} — <title>'") + if level == 2 and len(re.findall(r"(?m)^## ", text)) != 1: + raise LifecycleError("a rule section has exactly one level-two heading") + if level == 3 and (re.search(r"(?m)^## ", text) or len( + [m for m in re.finditer(r"(?m)^### ", text)]) != 1): + raise LifecycleError("a level-three rule section has exactly one heading and no ## heading") + if section.has_footer: + raise LifecycleError("the service writes lifecycle footers; do not include one") + if not 80 <= len(text) <= _MAX_SECTION_CHARS: + raise LifecycleError("section length is outside 80..16384 characters") + return section + + +def _new_page_text(page: str, title: str, today: str, template: Page | None) -> str: + tags = "[]" + if template is not None and template.frontmatter_field("tags") is not None: + tags = template.frontmatter_field("tags") + if not title or '"' in title: + raise LifecycleError("a new page needs a title without quotes") + return ( + "---\n" + f'title: "{title}"\n' + f"created: {today}\n" + f"updated: {today}\n" + "type: rule\n" + f"tags: {tags}\n" + "sources: []\n" + "---\n\n" + f"# {title}\n\n" + ) + + +def index_line(page: str, title: str) -> str: + return f"- [{title}]({PurePosixPath(page).name})\n" + + +def apply_operations( + files: Mapping[str, str], + operations: Iterable[KnowledgeOperation], + *, + release: str, + today: str, +) -> OperationsResult: + """Apply typed operations to ``files`` (knowledge-relative path -> text). + + Raises ``LifecycleError`` on the first invalid operation; nothing is + partially applied because the input mapping is never mutated. + """ + if not _DATE.fullmatch(today): + raise LifecycleError("today must be YYYY-MM-DD") + if not release or re.search(r'[\s"]', release): + raise LifecycleError("release must be a non-empty token") + work: dict[str, str] = dict(files) + original_ids = all_rule_ids(files) + tombstoned = all_tombstoned_ids(files) + added_ids: set[str] = set() + touched_pages: dict[str, list[str]] = {} # page -> sources before any change + created: list[str] = [] + touched_rules: list[str] = [] + purged: dict[str, list[dict]] = {} + + def page_of(path: str, *, create_title: str = "") -> Page: + if not _RULE_PAGE.fullmatch(PurePosixPath(path).name): + raise LifecycleError(f"not a rule page (rules.md / rules-<topic>.md): {path}") + repo_scope(path) + text = work.get(path) + if text is None: + if not create_title: + raise LifecycleError(f"page does not exist: {path}") + sibling = str(PurePosixPath(path).with_name("rules.md")) + template = Page.parse(work[sibling]) if sibling in work else None + index = str(PurePosixPath(path).with_name(INDEX_NAME)) + if index not in work: + raise LifecycleError(f"a new page's directory needs an {INDEX_NAME}: {index}") + text = _new_page_text(path, create_title, today, template) + work[path] = text + created.append(path) + work[index] = work[index].rstrip("\n") + "\n" + index_line(path, create_title) + page = Page.parse(text) + if page.frontmatter_field("type") != "rule": + raise LifecycleError(f"page is not type: rule: {path}") + touched_pages.setdefault(path, page.sources() if path not in created else []) + return page + + def fresh_id(rule_id: str) -> None: + if rule_id in original_ids or rule_id in added_ids: + raise LifecycleError(f"rule ID already exists in the tree: {rule_id}") + if rule_id in tombstoned: + raise LifecycleError(f"rule ID was purged and is reserved forever: {rule_id}") + added_ids.add(rule_id) + + def active(page: Page, rule_id: str, op: KnowledgeOperation) -> Section: + section = page.rule(rule_id) + footer = section.footer + if footer.status != "active": + raise LifecycleError(f"rule {rule_id} is not active") + if footer.protected and not op.allow_protected: + raise LifecycleError(f"rule {rule_id} is protected; only the human path may change it") + return section + + for op in operations: + if op.kind not in OP_KINDS: + raise LifecycleError(f"unknown operation kind: {op.kind}") + if op.kind == "add": + section = _check_section(op.section_markdown, op.rule_id) + fresh_id(op.rule_id) + for nested in section.nested_rule_ids: + if nested != op.rule_id: + fresh_id(nested) + page = page_of(op.page, create_title=op.page_title) + page = page.append_section(section.with_footer(Footer(status="active", since=release))) + work[op.page] = page.render() + touched_rules.append(op.rule_id) + elif op.kind == "edit_same_meaning": + page = page_of(op.page) + old = active(page, op.rule_id, op) + section = _check_section(op.section_markdown, op.rule_id, old.level) + if set(section.nested_rule_ids) != set(old.nested_rule_ids): + raise LifecycleError("an edit may not add or drop nested rule IDs") + trailing = old.text[len(old.text.rstrip("\n")):] + new = Section(section.text.rstrip("\n") + (trailing or "\n")) + if old.has_footer: + new = new.with_footer(old.footer) + work[op.page] = page.replace_section(op.rule_id, new).render() + touched_rules.append(op.rule_id) + elif op.kind == "replace": + if not op.new_rule_id or op.new_rule_id == op.rule_id: + raise LifecycleError("replace needs a different new_rule_id") + if not op.evidence: + raise LifecycleError("replace needs evidence") + page = page_of(op.page) + old = active(page, op.rule_id, op) + section = _check_section(op.section_markdown, op.new_rule_id) + fresh_id(op.new_rule_id) + for nested in section.nested_rule_ids: + if nested != op.new_rule_id: + fresh_id(nested) + retired = old.with_footer(Footer( + status="retired", since=old.footer.since, retired_at=release, + reason="superseded", evidence=op.evidence, + supersedes=old.footer.supersedes, # keep the chain A -> B -> C + superseded_by=op.new_rule_id, protected=old.footer.protected, + )) + work[op.page] = page.replace_section(op.rule_id, retired).render() + target = op.new_page or op.page + target_page = page_of(target, create_title=op.page_title) + successor = section.with_footer(Footer( + status="active", since=release, supersedes=op.rule_id)) + work[target] = target_page.append_section(successor).render() + touched_rules += [op.rule_id, op.new_rule_id] + elif op.kind == "retire": + if op.reason not in RETIRE_REASONS or op.reason == "superseded": + raise LifecycleError("retire reason must be upstream-removed, incorrect or duplicate; use replace for superseded") + if not op.evidence: + raise LifecycleError("retire needs evidence") + page = page_of(op.page) + old = active(page, op.rule_id, op) + retired = old.with_footer(Footer( + status="retired", since=old.footer.since, retired_at=release, + reason=op.reason, evidence=op.evidence, + supersedes=old.footer.supersedes, protected=old.footer.protected, + )) + work[op.page] = page.replace_section(op.rule_id, retired).render() + touched_rules.append(op.rule_id) + elif op.kind == "purge": + page = page_of(op.page) + old = page.rule(op.rule_id) + if old.footer.status != "retired": + raise LifecycleError(f"only a retired rule can be purged: {op.rule_id}") + if old.footer.protected and not op.allow_protected: + raise LifecycleError(f"rule {op.rule_id} is protected; only the human path may purge it") + if old.footer.retired_at == release: + raise LifecycleError("a rule is purged one full release after it was retired") + work[op.page] = page.replace_section(op.rule_id, None).render() + for rid in dict.fromkeys((op.rule_id, *old.nested_rule_ids)): + purged.setdefault(tombstones_path(op.page), []).append( + {"id": rid, "purged_at": release, "page": op.page}) + touched_rules.append(op.rule_id) + + for path, before_sources in touched_pages.items(): + page = Page.parse(work[path]) + page = page.with_frontmatter_field("updated", today) + page = page.with_sources(expected_sources(before_sources, page)) + work[path] = page.render() + for path, entries in purged.items(): + work[path] = render_tombstones(load_tombstones(work.get(path)) + entries) + for path in touched_pages: + over = page_over_capacity(work[path]) + if over: + raise LifecycleError(f"page full, add to a new rules-<topic>.md page instead: {path}: {over}") + + changed = {path: text for path, text in work.items() if files.get(path) != text} + return OperationsResult( + files=changed, + touched_rules=tuple(dict.fromkeys(touched_rules)), + created_pages=tuple(created), + ) diff --git a/.github/kb-gate/requirements.lock b/.github/kb-gate/requirements.lock new file mode 100644 index 00000000..895725e5 --- /dev/null +++ b/.github/kb-gate/requirements.lock @@ -0,0 +1,227 @@ +# GENERATED by tools/build_kb_gate_bundle.py --lock. Install with: +# pip install --require-hashes --no-deps -r .github/kb-gate/requirements.lock +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 +PyYAML==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 diff --git a/.github/kb-gate/signing.py b/.github/kb-gate/signing.py new file mode 100644 index 00000000..9c1934a6 --- /dev/null +++ b/.github/kb-gate/signing.py @@ -0,0 +1,118 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/signing.py -- do not edit here. +"""Ed25519 envelopes for knowledge-gate verdicts, outbox items and control records. + +An envelope is ``{"payload": <object>, "key_id": <str>, "signature": <b64>}``. +The signature covers the canonical JSON of the payload (sorted keys, compact +separators, UTF-8) prefixed with a purpose tag, so a verdict signature can never +be replayed as an outbox item or a control record and vice versa. + +Only ``cryptography`` beyond the standard library: the kb-gate verifier bundle +vendors this module and pins that dependency by hash. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, Ed25519PublicKey, +) + +PURPOSES = ("kb-gate-verdict", "kb-outbox-item", "kb-control", "kb-holds", "kb-ack") + + +class SignatureError(ValueError): + """An envelope is malformed, signed for another purpose, or forged.""" + + +def canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _message(purpose: str, payload: Any) -> bytes: + if purpose not in PURPOSES: + raise SignatureError(f"unknown signature purpose: {purpose}") + return purpose.encode("ascii") + b"\0" + canonical_json(payload) + + +def public_key_id(public: Ed25519PublicKey) -> str: + raw = public.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return "ed25519:" + hashlib.sha256(raw).hexdigest()[:16] + + +def public_key_text(public: Ed25519PublicKey) -> str: + """The one-line form committed as ``.github/kb-gate.pub``.""" + raw = public.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return "ed25519 " + base64.b64encode(raw).decode("ascii") + + +def load_public_key(text: str) -> Ed25519PublicKey: + parts = text.strip().split() + if len(parts) != 2 or parts[0] != "ed25519": + raise SignatureError("public key must be 'ed25519 <base64>'") + try: + return Ed25519PublicKey.from_public_bytes(base64.b64decode(parts[1], validate=True)) + except (ValueError, TypeError) as exc: + raise SignatureError("public key is not a valid Ed25519 key") from exc + + +def generate_private_key(path: str | Path) -> Ed25519PrivateKey: + """Create a new private key file readable only by its owner.""" + path = Path(path) + if path.exists(): + raise SignatureError(f"refusing to overwrite an existing key: {path}") + key = Ed25519PrivateKey.generate() + data = key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(data) + return key + + +def load_private_key(path: str | Path) -> Ed25519PrivateKey: + path = Path(path) + mode = path.stat().st_mode & 0o077 + if mode: + raise SignatureError(f"private key must not be readable by group/others: {path}") + key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(key, Ed25519PrivateKey): + raise SignatureError("private key is not Ed25519") + return key + + +def sign(purpose: str, payload: Any, key: Ed25519PrivateKey) -> dict: + signature = key.sign(_message(purpose, payload)) + return { + "purpose": purpose, + "payload": payload, + "key_id": public_key_id(key.public_key()), + "signature": base64.b64encode(signature).decode("ascii"), + } + + +def verify(purpose: str, envelope: Any, public: Ed25519PublicKey) -> Any: + """Return the payload of a valid envelope for ``purpose``; raise otherwise.""" + if not isinstance(envelope, dict) or set(envelope) != {"purpose", "payload", "key_id", "signature"}: + raise SignatureError("envelope must have exactly purpose, payload, key_id and signature") + if envelope["purpose"] != purpose: + raise SignatureError(f"envelope is signed for {envelope['purpose']!r}, not {purpose!r}") + if envelope["key_id"] != public_key_id(public): + raise SignatureError("envelope was signed by a different key") + try: + signature = base64.b64decode(envelope["signature"], validate=True) + public.verify(signature, _message(purpose, envelope["payload"])) + except (InvalidSignature, ValueError, TypeError) as exc: + raise SignatureError("signature does not verify") from exc + return envelope["payload"] diff --git a/.github/kb-gate/verdict.py b/.github/kb-gate/verdict.py new file mode 100644 index 00000000..59f44c4c --- /dev/null +++ b/.github/kb-gate/verdict.py @@ -0,0 +1,149 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/verdict.py -- do not edit here. +"""The kb-gate verdict: what the knowledge service signs and the verifier checks. + +A verdict binds a decision to ONE pull request state: + +* identity: repository, PR number, exact head SHA, nonce, issue time and a + 72-hour issue window (it must first verify within it); +* the complete patch manifest: every changed path with change type, pre-image + and post-image git blob IDs and file modes, compared independently of any + base SHA (so a queued PR is checked against what the queue actually applies); +* the per-block judgements for every rule and prose block L1 produced, the + change-set consistency judgement with the page hashes it saw, and the + context base those judgements were made against; +* upstream fact attestations (the verifier re-checks public upstreams); +* who decided: ``auto`` (L1 + L2 + consistency) or ``human-approved`` (with the + approving review IDs and reviewers). + +Standard library only (plus ``signing`` for the envelope): vendored by kb-gate. +""" + +from __future__ import annotations + +import hashlib +import re +import secrets +from dataclasses import dataclass +from typing import Any, Iterable, Mapping + +SCHEMA = "kb-gate-verdict/1" +ISSUE_WINDOW_SECONDS = 72 * 3600 +SOURCES = ("auto", "human-approved") +_SHA40 = re.compile(r"[0-9a-f]{40}") + + +class VerdictError(ValueError): + """A verdict is malformed or does not match what it claims to cover.""" + + +def git_blob_id(data: bytes | str | None) -> str: + """The git object ID of a blob with this content ("" for an absent file).""" + if data is None: + return "" + if isinstance(data, str): + data = data.encode("utf-8") + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + + +@dataclass(frozen=True) +class ManifestEntry: + path: str + status: str # A | M | D + old_blob: str + new_blob: str + old_mode: str + new_mode: str + + def to_dict(self) -> dict: + return {"path": self.path, "status": self.status, "old_blob": self.old_blob, + "new_blob": self.new_blob, "old_mode": self.old_mode, "new_mode": self.new_mode} + + +def manifest_from_files(base: Mapping[str, str], head: Mapping[str, str], *, + prefix: str = "knowledge/") -> list[dict]: + """Patch manifest for a change expressed as knowledge-relative text maps.""" + entries = [] + for rel in sorted(set(base) | set(head)): + before, after = base.get(rel), head.get(rel) + if before == after: + continue + status = "A" if before is None else "D" if after is None else "M" + entries.append(ManifestEntry( + path=prefix + rel, status=status, + old_blob=git_blob_id(before), new_blob=git_blob_id(after), + old_mode="" if before is None else "100644", + new_mode="" if after is None else "100644", + ).to_dict()) + return entries + + +def canonical_manifest(entries: Iterable[Mapping[str, Any]]) -> list[dict]: + """Sorted, field-complete manifest; the comparison form (no base SHA).""" + out = [] + for entry in entries: + missing = {"path", "status", "old_blob", "new_blob", "old_mode", "new_mode"} - set(entry) + if missing: + raise VerdictError(f"manifest entry lacks {sorted(missing)}") + out.append({key: str(entry[key]) for key in + ("path", "status", "old_blob", "new_blob", "old_mode", "new_mode")}) + return sorted(out, key=lambda item: item["path"]) + + +def build_verdict(*, repository: str, pr: int, head_sha: str, context_base_sha: str, + manifest: list[dict], blocks: list[dict], consistency: list[dict], + release: str, upstream: dict, facts: list[dict], models: dict, + source: str = "auto", review_ids: tuple[int, ...] = (), + reviewers: tuple[str, ...] = (), issued_at: float) -> dict: + if source not in SOURCES: + raise VerdictError(f"source must be one of {SOURCES}") + if not _SHA40.fullmatch(head_sha) or not _SHA40.fullmatch(context_base_sha): + raise VerdictError("head_sha and context_base_sha must be 40-character hex SHAs") + if source == "human-approved" and not (review_ids and reviewers): + raise VerdictError("a human-approved verdict names its approving reviews") + for block in blocks: + if block.get("verdict") != "pass": + raise VerdictError(f"only passing blocks are signed: {block.get('block_id')}") + for item in consistency: + if item.get("verdict") != "consistent": + raise VerdictError("only a consistent change set is signed") + return { + "schema": SCHEMA, + "repository": repository, + "pr": int(pr), + "head_sha": head_sha, + "context_base_sha": context_base_sha, + "issued_at": float(issued_at), + "valid_until": float(issued_at) + ISSUE_WINDOW_SECONDS, + "nonce": secrets.token_hex(16), + "source": source, + "review_ids": [int(value) for value in review_ids], + "reviewers": list(reviewers), + "release": release, + "upstream": upstream, + "manifest": canonical_manifest(manifest), + "blocks": sorted(blocks, key=lambda b: b["block_id"]), + "consistency": consistency, + "facts": facts, + "models": models, + } + + +def check_binding(verdict: Mapping[str, Any], *, repository: str, pr: int, head_sha: str, + now: float) -> None: + """Identity half of verification: the verdict names THIS PR state and is + within its issue window. Replaying a verdict on another PR, another head, + or after the window fails here.""" + if verdict.get("schema") != SCHEMA: + raise VerdictError("unknown verdict schema") + if verdict.get("repository") != repository or int(verdict.get("pr", -1)) != int(pr): + raise VerdictError("verdict is for another repository or pull request") + if verdict.get("head_sha") != head_sha: + raise VerdictError("verdict is for another head commit") + if not float(verdict.get("issued_at", 0)) <= now <= float(verdict.get("valid_until", 0)): + raise VerdictError("verdict is outside its issue window") + if verdict.get("source") not in SOURCES: + raise VerdictError("verdict source is invalid") + + +def manifests_equal(signed: Iterable[Mapping[str, Any]], actual: Iterable[Mapping[str, Any]]) -> bool: + return canonical_manifest(signed) == canonical_manifest(actual) diff --git a/.github/workflows/kb-gate.yml b/.github/workflows/kb-gate.yml new file mode 100644 index 00000000..85dcfb15 --- /dev/null +++ b/.github/workflows/kb-gate.yml @@ -0,0 +1,99 @@ +name: kb-gate + +# The required check for knowledge pull requests (doc/features/knowledge-service.md). +# +# Two stages, one check name: +# * precheck (pull_request_target / issue_comment / workflow_dispatch): verifies +# the PR against current main and posts the `kb-gate` commit STATUS on the PR +# head. It only decides whether the PR may enter the merge queue. +# * merge group (merge_group): the job named `kb-gate` verifies every segment of +# the group on the exact commit that would land. This decides what lands. +# +# Security: the precheck runs with the BASE branch's workflow and bundle +# (pull_request_target), reads the PR only as git objects, never checks it out, +# never installs or executes anything from it, and holds no secret (the repository +# is public, so PR refs are fetched anonymously; the token only reads the API and +# posts the status). Both jobs +# run the verifier ONLY from .github/kb-gate/ (pinned, hash-locked deps). + +on: + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled, edited, ready_for_review, converted_to_draft] + issue_comment: + types: [created, edited] + workflow_dispatch: + inputs: + pr: + description: PR number to (re)check + required: true + merge_group: + types: [checks_requested] + +permissions: + contents: read + +jobs: + precheck: + if: >- + github.event_name == 'pull_request_target' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + contains(github.event.comment.body, '<!-- kb-gate:verdict:v1 -->')) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + statuses: write + concurrency: + group: kb-gate-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned verifier + run: | + cd .github/kb-gate && sha256sum --check --strict MANIFEST.sha256 + python -m venv "$RUNNER_TEMP/kb-gate" + "$RUNNER_TEMP/kb-gate/bin/pip" install --quiet --require-hashes --no-deps -r requirements.lock + - name: Verify and post the kb-gate status + env: + GITHUB_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number || github.event.issue.number || inputs.pr }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + "$RUNNER_TEMP/kb-gate/bin/python" -E -s .github/kb-gate/kb_gate.py pr \ + --repository "$GITHUB_REPOSITORY" --pr "$PR" --post-status --target-url "$RUN_URL" + + kb-gate: + if: github.event_name == 'merge_group' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned verifier + run: | + cd .github/kb-gate && sha256sum --check --strict MANIFEST.sha256 + python -m venv "$RUNNER_TEMP/kb-gate" + "$RUNNER_TEMP/kb-gate/bin/pip" install --quiet --require-hashes --no-deps -r requirements.lock + - name: Verify every segment of the merge group + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + BASE_REF: ${{ github.event.merge_group.base_ref }} + run: | + "$RUNNER_TEMP/kb-gate/bin/python" -E -s .github/kb-gate/kb_gate.py merge-group \ + --repository "$GITHUB_REPOSITORY" --head-sha "$HEAD_SHA" --base-ref "${BASE_REF#refs/heads/}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c65dc5c..e22e731f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,7 @@ name: tests on: push: pull_request: + merge_group: workflow_dispatch: permissions: @@ -183,6 +184,12 @@ jobs: - name: SPEC coverage and freshness run: python tools/check_spec_freshness.py --strict + # A reminder, not a gate: a stale bundle keeps kb-gate at its previous + # strictness. Regenerating it is a .github/** change (CODEOWNERS review). + - name: kb-gate bundle matches its source + continue-on-error: true + run: python tools/build_kb_gate_bundle.py --check + # Written, tested, and until now invoked only from prose (AGENTS.md, DEVELOPMENT.md). - name: Knowledge linters run: | diff --git a/doc/architecture/SPEC/knowledge_service.md b/doc/architecture/SPEC/knowledge_service.md index 9917de17..eec2f95c 100644 --- a/doc/architecture/SPEC/knowledge_service.md +++ b/doc/architecture/SPEC/knowledge_service.md @@ -47,3 +47,26 @@ PR, head or window. Standard library only; vendored by kb-gate. ## 2026-09-28 T1 索引修复 L1 的索引检查允许为目录中已存在但未列入索引的页面补链接（巡检 T1 修复），新页面仍必须被链接。 + +## 2026-09-28 kb-gate 验证器 +`gate_verifier` 是仓库端必需检查 `kb-gate`。`tools/build_kb_gate_bundle.py` 把它和 +`l1`、`lifecycle`、`ops`、`signing`、`verdict` 原样复制到 `.github/kb-gate/`（附 +`requirements.lock` 按哈希固定依赖，`MANIFEST.sha256` 为包内文件哈希），工作流只从该目录运行。 +- PR 阶段（`pull_request_target`/带判定标记的 `issue_comment`/手动）：用 git 对象读取 PR（从不检出 + 或执行），在 `merge-tree` 结果上跑 L1，把 `kb-gate` 状态发布到**已验证的**那个 head。 +- merge group 阶段：`main..head` 的 first-parent 链每段必须是双亲合并提交；第二父按 open PR 的 + head 唯一反查；每个触碰 `knowledge/` 的段以其第一父为有效 base 独立验证，最后在整棵落地树上跑 L1。 +- 知识变更通过条件：签名有效且绑定本仓库/PR/head、在签发有效期内；清单与实际变更（与 base 无关） + 相等；`auto` 要求块表与 L1 分块完全一致且全部 pass、一致性判定覆盖相同 owner 目录且页面哈希与落地树 + 相同、`context_base_sha..有效 base` 未改动上下文集合；`human-approved` 要求绑定的审阅仍为 + APPROVED、针对当前 head、由 `.github/kb-gate/` 的 CODEOWNERS（知识维护者）之一给出，且标签仍在。 + 另需无 `kb:hold`、签名暂停清单新鲜（≤10 分钟）且未命中；清单缺失/伪造/过期一律失败。 + `human-approved` 豁免自动合并白名单（`knowledge/tools`、`skills`、脚本、混合 PR），页面生命周期 L1 只作用于 + 受治理页面，其余 L1 不豁免； + 本变更退役/删除的规则 ID 在落地树中不得被引用（排在后面的 PR 或 main 可能新增了引用）。 + 暂停清单按范围匹配：`repos/<r>/` 对应仓库名 `<r>`，`general/` 对应 `general`。 +- 不变量：失败即关闭——缺公钥、缺 `holds_url`、无法归属的 merge group、带上游事实证明的判定都失败； + 未触碰 `knowledge/` 的变更不读取任何判定或暂停清单，总是通过。git < 2.38 时 PR 阶段用临时索引做 + 只接受单侧改动的三方合并（更严格，不更宽松）。 +- 测试：`test_kb_gate_verifier.py`（重放/过期/伪造、清单漏项、块表不全、混合路径与可执行位、暂停清单、 + 上下文失效、状态发布、人工审批撤销、merge group 逐段与前序同目录 PR、验证包新鲜与自包含、独立运行）。 diff --git a/doc/features/knowledge-service.md b/doc/features/knowledge-service.md index 128faac9..5bf87c69 100644 --- a/doc/features/knowledge-service.md +++ b/doc/features/knowledge-service.md @@ -64,3 +64,28 @@ T1/T2/T3 与 purge，每个规则页一个变更集，全部经过质量门。 评审服务读取知识：将 `KNOWLEDGE_ROOT` 指向 `$KB_STATE_DIR/active`。 + +## 仓库端门禁 `kb-gate` + +必需检查 `kb-gate`（`.github/workflows/kb-gate.yml`）只从受 CODEOWNERS 保护的验证包 +`.github/kb-gate/` 运行，从不导入 `src/`、`tools/`、`knowledge/tools/`。 + +- **PR 预检**：`pull_request_target`、带判定标记的评论或手动触发；在当前 `main` 上验证后， + 把 `kb-gate` 状态发布到 PR head。只决定能否入队。 +- **merge group**：名为 `kb-gate` 的作业逐段验证将要落地的提交；失败则 PR 被移出队列。 +- 未触碰 `knowledge/` 的 PR 总是通过；知识 PR 需要有效签名判定、清单一致、L1 通过、 + 暂停清单新鲜且未命中（见 SPEC `knowledge_service.md` 的 kb-gate 验证器一节）。 + +修改验证逻辑后重新生成验证包（属于 `.github/**` 改动，需要知识维护者审批）： + +```bash +python tools/build_kb_gate_bundle.py # 重新生成 +python tools/build_kb_gate_bundle.py --check # CI 提醒（非阻塞） +python tools/build_kb_gate_bundle.py --lock # 从 PyPI 重新按哈希固定依赖 +``` + +切换前的负责人步骤：提交 `.github/kb-gate.pub`（`kb keygen` 的公钥）；在 +`.github/kb-gate/config.json` 填入暂停清单的公开 HTTPS 地址（`$KB_STATE_DIR/public/holds.json`）； +在 `.github/CODEOWNERS` 中再加入至少一名知识维护者；仓库 admin 开启合并队列（merge commit） +与 ruleset（必需检查 `kb-gate` 限定 GitHub Actions 来源、Code Owner 审阅、新推送需重新批准）。 +在此之前，所有触碰 `knowledge/` 的 PR 都会被 `kb-gate` 拒绝（失败即关闭），代码 PR 不受影响。 diff --git a/src/infermatrix_copilot/knowledge_service/gate_verifier.py b/src/infermatrix_copilot/knowledge_service/gate_verifier.py new file mode 100644 index 00000000..d90224a5 --- /dev/null +++ b/src/infermatrix_copilot/knowledge_service/gate_verifier.py @@ -0,0 +1,691 @@ +"""kb-gate: the repository-side verifier for knowledge pull requests. + +It is the required check ``kb-gate`` and runs in two places under one name: + +* PR stage (``pull_request_target`` / ``issue_comment`` / dispatch): verifies + the PR against the current base and posts a ``kb-gate`` commit status on the + PR head. It only decides whether a PR may ENTER the merge queue. +* merge group stage (``merge_group``): verifies every segment of the group on + the exact commit that would land and exits non-zero on any failure. This is + what decides whether anything lands. + +A change that does not touch ``knowledge/`` passes. A knowledge change passes +only when all of these hold for it: + +1. L1: whitelisted paths only, regular files only, no new tree issues, on the + PR's own change AND on the tree that would land; +2. a valid Ed25519-signed verdict comment for this repository, PR and exact + head SHA, inside its issue window (the comment's author is irrelevant); +3. the signed patch manifest equals the change as applied (base independent); +4. ``auto``: the signed block table equals the blocks L1 derives, every block + passed, the consistency judgements cover the same owner directories and + their page hashes match the tree that would land; no page in the context + set changed between the judged base and the effective base; + ``human-approved``: every bound review is still APPROVED on this head, one + is by a knowledge maintainer (a CODEOWNERS owner of ``.github/kb-gate/``), + and ``kb:human-approved`` is set; +5. no ``kb:hold`` label and not held by the service's signed hold list (a + missing, forged or stale hold list fails closed). + +PR content is only ever READ through git objects; nothing from the PR is +executed or installed. This module is vendored into ``.github/kb-gate/`` by +``tools/build_kb_gate_bundle.py`` and runs from there, so it imports nothing +outside the standard library and its vendored siblings. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Iterable, Mapping + +try: + from .l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree + from .lifecycle import LifecycleError, Page + from .ops import all_rule_ids, repo_scope + from .signing import SignatureError, load_public_key, verify + from .verdict import VerdictError, check_binding, manifests_equal +except ImportError: # pragma: no cover - vendored verifier bundle + from l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree # type: ignore[no-redef] + from lifecycle import LifecycleError, Page # type: ignore[no-redef] + from ops import all_rule_ids, repo_scope # type: ignore[no-redef] + from signing import SignatureError, load_public_key, verify # type: ignore[no-redef] + from verdict import VerdictError, check_binding, manifests_equal # type: ignore[no-redef] + +STATUS_CONTEXT = "kb-gate" +VERDICT_MARKER = "<!-- kb-gate:verdict:v1 -->" +HOLD_LABEL = "kb:hold" +HUMAN_LABEL = "kb:human-approved" +HOLDS_MAX_AGE = 10 * 60 +MAINTAINER_PATH = ".github/kb-gate/kb_gate.py" +WHITELIST_CODES = ("path_not_whitelisted", "mode_not_regular") +KNOWLEDGE_PREFIX = "knowledge/" +GOVERNED_PREFIXES = ("knowledge/repos/", "knowledge/general/") +GOVERNED_SUFFIXES = (".md", ".yaml") +EXTERNAL_REF_DIRS = ("skills/", "plugins/", "adapters/", "doc/", "playbooks/") +EXTERNAL_SUFFIXES = (".md", ".yaml", ".yml", ".json", ".txt") +_ZERO = "0" * 40 + + +class GateError(RuntimeError): + """The verifier could not establish a fact it needs (fails closed).""" + + +# -- git (objects only; nothing from the PR is ever checked out or executed) --- + +class Git: + def __init__(self, cwd: str | Path): + self.cwd = str(cwd) + + def run(self, *args: str, input: bytes | None = None, ok: tuple[int, ...] = (0,)) -> bytes: + proc = subprocess.run(["git", *args], cwd=self.cwd, input=input, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode not in ok: + raise GateError(f"git {' '.join(args)} failed: {proc.stderr.decode(errors='replace')[:400]}") + return proc.stdout + + def rev(self, name: str) -> str: + return self.run("rev-parse", "--verify", "--quiet", f"{name}^{{commit}}").decode().strip() + + def tree_of(self, name: str) -> str: + return self.run("rev-parse", "--verify", f"{name}^{{tree}}").decode().strip() + + def parents(self, sha: str) -> list[str]: + return self.run("rev-list", "--parents", "-n", "1", sha).decode().split()[1:] + + def is_ancestor(self, older: str, newer: str) -> bool: + proc = subprocess.run(["git", "merge-base", "--is-ancestor", older, newer], cwd=self.cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=False) + if proc.returncode not in (0, 1): + raise GateError(f"cannot relate {older[:12]} and {newer[:12]}: " + f"{proc.stderr.decode(errors='replace')[:200]}") + return proc.returncode == 0 + + def merge_base(self, a: str, b: str) -> str: + return self.run("merge-base", a, b).decode().strip() + + def first_parent_chain(self, base: str, head: str) -> list[str]: + return self.run("rev-list", "--first-parent", "--reverse", f"{base}..{head}").decode().split() + + def merge_tree(self, base: str, head: str) -> str: + """The tree a merge of ``head`` into ``base`` would produce (PR stage).""" + proc = subprocess.run(["git", "merge-tree", "--write-tree", base, head], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode != 0 and b"--write-tree" in proc.stderr: # git < 2.38: no --write-tree + return self._index_merge(base, head) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().split()[0] + + def _index_merge(self, base: str, head: str) -> str: + """Only merges where each path changed on at most one side; anything + else is reported as not merging cleanly (stricter, never looser).""" + import tempfile + + with tempfile.TemporaryDirectory() as scratch: + env = {**os.environ, "GIT_INDEX_FILE": str(Path(scratch) / "index")} + for args in (["read-tree", "-i", "-m", self.merge_base(base, head), base, head], ["write-tree"]): + proc = subprocess.run(["git", *args], cwd=self.cwd, env=env, check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().strip() + + def raw_diff(self, old: str, new: str) -> list[dict]: + """Every changed path as a manifest entry (renames are D + A).""" + out = self.run("diff", "--raw", "-z", "--no-renames", "--no-abbrev", old, new).split(b"\0") + entries: list[dict] = [] + index = 0 + while index < len(out) - 1: + meta = out[index].decode() + path = out[index + 1].decode("utf-8", "replace") + index += 2 + if not meta.startswith(":"): + raise GateError(f"unexpected git diff output: {meta!r}") + old_mode, new_mode, old_blob, new_blob, status = meta[1:].split() + entries.append({ + "path": path, "status": status[0], + "old_blob": "" if old_blob == _ZERO else old_blob, + "new_blob": "" if new_blob == _ZERO else new_blob, + "old_mode": "" if old_mode == "000000" else old_mode, + "new_mode": "" if new_mode == "000000" else new_mode, + }) + return entries + + def changed_names(self, old: str, new: str) -> list[str]: + out = self.run("diff", "--name-only", "-z", "--no-renames", old, new, "--", "knowledge/") + return [name.decode("utf-8", "replace") for name in out.split(b"\0") if name] + + def texts(self, rev: str, prefixes: tuple[str, ...], suffixes: tuple[str, ...]) -> dict[str, str]: + listing = self.run("ls-tree", "-r", "-z", "--full-tree", rev).split(b"\0") + wanted: list[tuple[str, str]] = [] + for entry in listing: + if not entry: + continue + meta, _, raw_path = entry.partition(b"\t") + mode, kind, oid = meta.decode().split() + name = raw_path.decode("utf-8", "replace") + if kind == "blob" and mode == "100644" and name.startswith(prefixes) and name.endswith(suffixes): + wanted.append((name, oid)) + if not wanted: + return {} + out = self.run("cat-file", "--batch", input="".join(f"{oid}\n" for _, oid in wanted).encode()) + texts: dict[str, str] = {} + cursor = 0 + for name, _oid in wanted: + header_end = out.index(b"\n", cursor) + size = int(out[cursor:header_end].split()[2]) + body = out[header_end + 1: header_end + 1 + size] + cursor = header_end + 1 + size + 1 + try: + texts[name] = body.decode("utf-8") + except UnicodeDecodeError: + continue + return texts + + def knowledge_files(self, rev: str) -> dict[str, str]: + files = self.texts(rev, GOVERNED_PREFIXES, GOVERNED_SUFFIXES) + return {name[len(KNOWLEDGE_PREFIX):]: text for name, text in files.items()} + + def external_texts(self, rev: str) -> dict[str, str]: + return self.texts(rev, EXTERNAL_REF_DIRS, EXTERNAL_SUFFIXES) + + def show(self, rev: str, path: str) -> str | None: + proc = subprocess.run(["git", "show", f"{rev}:{path}"], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + return proc.stdout.decode("utf-8", "replace") if proc.returncode == 0 else None + + +# -- GitHub REST (read, plus the PR-stage status) -------------------------------- + +class GitHub: + def __init__(self, repository: str, token: str, api: str = "https://api.github.com"): + self.repository, self._token, self._api = repository, token, api.rstrip("/") + + def _request(self, method: str, path: str, body: dict | None = None) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(self._api + path, data=data, method=method, headers={ + "Accept": "application/vnd.github+json", "Authorization": f"Bearer {self._token}", + "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "kb-gate"}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read() or b"null") + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"GitHub {method} {path} failed: {exc}") from exc + + def get(self, path: str) -> Any: + return self._request("GET", path) + + def get_all(self, path: str, *, pages: int = 10) -> list: + out: list = [] + sep = "&" if "?" in path else "?" + for page in range(1, pages + 1): + batch = self.get(f"{path}{sep}per_page=100&page={page}") + out.extend(batch) + if len(batch) < 100: + break + return out + + def post(self, path: str, body: dict) -> Any: + return self._request("POST", path, body) + + +# -- inputs from the protected checkout ------------------------------------------- + +def fetch_holds(url: str) -> Any: + if not url: + raise GateError("no hold-list URL is configured (.github/kb-gate/config.json)") + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "kb-gate"}), + timeout=20) as response: + return json.loads(response.read()) + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"hold list unreachable: {exc}") from exc + + +def verify_holds(envelope: Any, public_key, *, now: float) -> dict: + holds = verify("kb-holds", envelope, public_key) + if not 0 <= now - float(holds["issued_at"]) <= HOLDS_MAX_AGE: + raise GateError("hold list is stale (the knowledge service may be down)") + return holds + + +def parse_codeowners(text: str | None) -> list[tuple[str, tuple[str, ...]]]: + rules = [] + for line in (text or "").splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + pattern, *owners = line.split() + rules.append((pattern, tuple(owner.lstrip("@").lower() for owner in owners))) + return rules + + +def _codeowner_match(pattern: str, path: str) -> bool: + anchored = pattern.startswith("/") + pattern = pattern.lstrip("/") + if pattern.endswith("/"): + pattern += "**" + candidates = [path] if anchored or "/" in pattern.rstrip("*") else \ + [path] + [path.split("/", i)[-1] for i in range(1, path.count("/") + 1)] + for candidate in candidates: + if fnmatch.fnmatchcase(candidate, pattern) or fnmatch.fnmatchcase(candidate, pattern + "/**") \ + or (pattern.endswith("/**") and candidate.startswith(pattern[:-2])): + return True + return False + + +def code_owners(rules: list[tuple[str, tuple[str, ...]]], path: str) -> tuple[str, ...]: + """Owners of ``path``: the LAST matching line wins. Team entries never + match a person here (their membership is not readable with the job token), + so a team-only path cannot be human-approved through this verifier.""" + owners: tuple[str, ...] = () + for pattern, names in rules: + if _codeowner_match(pattern, path): + owners = names + return tuple(name for name in owners if "/" not in name) + + +# -- the verification of one change ------------------------------------------------ + +@dataclass +class Context: + git: Git + github: Any + repository: str + public_key: Any # None when .github/kb-gate.pub is missing + holds_loader: Callable[[], Any] + codeowners: list[tuple[str, tuple[str, ...]]] + now: float + _holds: dict | None = None + + def holds(self) -> dict: + if self._holds is None: + self._holds = verify_holds(self.holds_loader(), self.public_key, now=self.now) + return self._holds + + +def governed(path: str) -> bool: + """A page the knowledge lifecycle governs (what ``knowledge_files`` reads).""" + return path.startswith(GOVERNED_PREFIXES) and path.endswith(GOVERNED_SUFFIXES) + + +def touches_knowledge(entries: Iterable[Mapping[str, str]]) -> bool: + return any(entry["path"].startswith(KNOWLEDGE_PREFIX) for entry in entries) + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def verdict_envelopes(comments: Iterable[Mapping[str, Any]]) -> list[Any]: + out = [] + for comment in comments: + body = str(comment.get("body") or "") + if VERDICT_MARKER not in body: + continue + rest = body.split(VERDICT_MARKER, 1)[1] + start = rest.find("```json") + if start < 0: + continue + rest = rest[start + len("```json"):] + end = rest.find("```") + try: + out.append(json.loads(rest[:end] if end >= 0 else rest)) + except ValueError: + continue + return out + + +def find_verdict(ctx: Context, pr: int, head_sha: str) -> tuple[dict | None, list[str]]: + """The newest valid verdict for exactly this PR head, and why others failed.""" + comments = ctx.github.get_all(f"/repos/{ctx.repository}/issues/{pr}/comments") + valid, rejected = [], [] + for envelope in verdict_envelopes(comments): + try: + verdict = verify("kb-gate-verdict", envelope, ctx.public_key) + check_binding(verdict, repository=ctx.repository, pr=pr, head_sha=head_sha, now=ctx.now) + valid.append(verdict) + except (SignatureError, VerdictError, KeyError, TypeError, ValueError) as exc: + rejected.append(str(exc)) + if not valid: + return None, rejected + return max(valid, key=lambda v: float(v["issued_at"])), rejected + + +def _owner_dir(path: str) -> str: + return str(PurePosixPath(path).parent) + + +def _scope_root(path: str) -> str: + parts = PurePosixPath(path).parts + return "/".join(parts[:2]) if parts[:1] == ("repos",) else parts[0] + + +def context_paths(touched: set[str], owner_dirs: set[str], post: Mapping[str, str], + base: Mapping[str, str]) -> Callable[[str], bool]: + """The pages whose change since the judged base invalidates L2: the touched + pages, every page in their owner directories, the scope's _routes.yaml and + _index.md files (scope root and owner directories), and pages holding rules + the change supersedes.""" + roots = {_scope_root(path) for path in touched} + dirs = owner_dirs | {_owner_dir(path) for path in touched} + superseded: set[str] = set() + for path in touched: + text = post.get(path) + if not text or not path.endswith(".md"): + continue + try: + sections = Page.parse(text).rules() + except (LifecycleError, ValueError): + continue + for section in sections: + try: + target = section.footer.supersedes + except LifecycleError: + continue + for rule_id in [t.strip() for t in target.split(",") if t.strip()]: + superseded.update(p for p, t in base.items() + if p.endswith(".md") and rule_id in all_rule_ids({p: t})) + + def relevant(path: str) -> bool: + name = PurePosixPath(path).name + parent = _owner_dir(path) + return (path in touched or path in superseded or parent in dirs + or (name in ("_routes.yaml", "_index.md") and parent in roots | dirs)) + return relevant + + +def verify_change(ctx: Context, *, pr: dict, head_sha: str, pre: str, post: str, final: str, + effective_base: str, stage: str) -> list[str]: + """Problems with one PR's knowledge change (empty when it passes). + + ``pre``/``post`` are the commits (or trees) around this PR's change as it + is applied; ``final`` is the tree that would land; ``effective_base`` is + the state the change is applied on (current main, or the first parent of + the merge-group segment).""" + number = int(pr["number"]) + entries = ctx.git.raw_diff(pre, post) + if not touches_knowledge(entries): + return [] + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + problems: list[str] = [] + labels = {str(label.get("name")) for label in pr.get("labels") or []} + if pr.get("state") != "open": + problems.append(f"PR #{number} is not open") + if str((pr.get("head") or {}).get("sha")) != head_sha: + problems.append(f"PR #{number} head moved from {head_sha[:12]}") + if stage == "merge_group" and pr.get("draft"): + problems.append(f"PR #{number} is a draft (paused)") + if HOLD_LABEL in labels: + problems.append(f"PR #{number} carries {HOLD_LABEL}") + + pre_files, post_files = ctx.git.knowledge_files(pre), ctx.git.knowledge_files(post) + final_files = ctx.git.knowledge_files(final) + changes = [Change(e["path"], e["status"] if e["status"] in "AMD" else "M", e["old_mode"], e["new_mode"]) + for e in entries] + for e in entries: + if e["status"] not in ("A", "M", "D"): + problems.append(f"{e['path']}: change type {e['status']} is not allowed") + touched = {e["path"][len(KNOWLEDGE_PREFIX):] for e in entries if e["path"].startswith(KNOWLEDGE_PREFIX)} + + try: + holds = ctx.holds() + if holds.get("global"): + problems.append("the knowledge service holds every repository (global pause)") + if number in {int(p) for p in holds.get("prs") or []}: + problems.append(f"PR #{number} is on the service's hold list") + held = set(holds.get("repos") or []) + # a scope is its repository name, or "general" for knowledge/general/ + scopes = {repo_scope(p).split("/")[-1] for p in touched if p.startswith(("repos/", "general/"))} + for scope in sorted(scopes & held): + problems.append(f"repository {scope} is paused by the knowledge service") + except (GateError, SignatureError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems.append(f"hold list: {exc}") + + verdict, rejected = find_verdict(ctx, number, head_sha) + if verdict is None: + detail = f" (rejected: {'; '.join(rejected[:3])})" if rejected else "" + problems.append(f"no valid signed verdict for PR #{number} at {head_sha[:12]}{detail}") + # still report L1, so the author sees every structural problem at once + result = check_changeset(pre_files, post_files, changes, external_texts={}) + return problems + [f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues] + + if not manifests_equal(verdict.get("manifest") or [], entries): + problems.append("the signed patch manifest does not match the change as applied") + if verdict.get("facts"): + problems.append("the verdict carries upstream fact attestations this verifier cannot re-check") + + human = verdict["source"] == "human-approved" + # a maintainer's approval replaces the auto-merge whitelist (knowledge/tools, + # skills, scripts, mixed PRs) and L2, never the rest of L1: non-governed + # paths are not pages, so the page lifecycle checks run on governed ones only + l1_changes = [c for c in changes if governed(c.path)] if human else changes + result = check_changeset(pre_files, post_files, l1_changes, + external_texts=ctx.git.external_texts(final), + release=str(verdict.get("release") or "")) + waived = WHITELIST_CODES if human else () + problems.extend(f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues if i.code not in waived) + # what this change retires or purges must not be cited anywhere in the tree + # that would land (a PR queued behind it, or main, may have added a citation) + gone = set(result.retired) | set(result.purged) + if gone: + try: + problems.extend(f"L1 {i.code} {i.path} {i.detail} (in the tree that would land)" + for i in dangling_refs(final_files, gone)) + except (LifecycleError, ValueError) as exc: + problems.append(f"L1 the tree that would land does not parse: {exc}") + + if verdict["source"] == "auto": + if result.external_refs: + problems.append("retired rules are still cited outside knowledge/ (needs a companion PR)") + signed_blocks = verdict.get("blocks") or [] + if {b.block_id for b in result.blocks} != {b.get("block_id") for b in signed_blocks}: + problems.append("the signed block table does not cover exactly the changed blocks") + if any(b.get("verdict") != "pass" for b in signed_blocks): + problems.append("a signed block did not pass") + owner_dirs = {_owner_dir(b.path) for b in result.blocks} + consistency = verdict.get("consistency") or [] + if {c.get("owner_dir") for c in consistency} != owner_dirs: + problems.append("the consistency judgement does not cover exactly the changed owner directories") + for item in consistency: + directory = item.get("owner_dir") + actual = {p: _sha256(t) for p, t in final_files.items() + if _owner_dir(p) == directory and p.endswith(".md")} + if item.get("verdict") != "consistent": + problems.append(f"{directory}: consistency was not judged consistent") + if item.get("pages") != actual: + problems.append(f"{directory}: pages changed since the consistency judgement") + base_sha = str(verdict["context_base_sha"]) + try: + if not ctx.git.is_ancestor(base_sha, effective_base): + problems.append("the judged context base is not an ancestor of the effective base") + else: + relevant = context_paths(touched, owner_dirs, post_files, pre_files) + moved = [n[len(KNOWLEDGE_PREFIX):] for n in ctx.git.changed_names(base_sha, effective_base)] + stale = sorted(p for p in moved if relevant(p)) + if stale: + problems.append("context changed since the verdict was judged: " + ", ".join(stale[:5])) + except GateError as exc: + problems.append(f"context base: {exc}") + else: + problems.extend(_human_approval_problems(ctx, pr, verdict, head_sha, labels)) + return problems + + +def _human_approval_problems(ctx: Context, pr: dict, verdict: dict, head_sha: str, + labels: set[str]) -> list[str]: + problems = [] + if HUMAN_LABEL not in labels: + problems.append(f"{HUMAN_LABEL} was removed") + number = int(pr["number"]) + reviewers = {str(r).lower() for r in verdict.get("reviewers") or []} + approvers = set() + for review_id in verdict.get("review_ids") or []: + review = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}/reviews/{int(review_id)}") + login = str((review.get("user") or {}).get("login") or "").lower() + if review.get("state") != "APPROVED": + problems.append(f"review {review_id} is no longer APPROVED") + elif review.get("commit_id") != head_sha: + problems.append(f"review {review_id} approved another head") + elif login not in reviewers: + problems.append(f"review {review_id} is by someone the verdict does not name") + else: + approvers.add(login) + # knowledge maintainers = the CODEOWNERS of the gate itself. Governed pages + # deliberately have no code owners (that would force a review on every auto + # PR); paths outside the whitelist get GitHub's native code-owner review. + maintainers = set(code_owners(ctx.codeowners, MAINTAINER_PATH)) + if not approvers & maintainers: + problems.append("no valid approving review by a knowledge maintainer (CODEOWNERS of .github/kb-gate/)") + return problems + + +def tree_problems(git: Git, base: str, final: str) -> list[str]: + """L1 on the tree that would land: no tree-level issue that base lacks.""" + before = {(i.code, i.detail) for i in check_tree(git.knowledge_files(base))} + return [f"L1 {i.code} {i.path} {i.detail}".rstrip() + for i in check_tree(git.knowledge_files(final)) if (i.code, i.detail) not in before] + + +# -- the two stages ----------------------------------------------------------------- + +def verify_pr(ctx: Context, number: int, *, base_ref: str = "main", + remote: str = "origin") -> tuple[str, list[str]]: + """(the head SHA verified, its problems). The status goes on THAT head, so + a push during verification can only leave the new head without a status.""" + pr = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}") + head = str(pr["head"]["sha"]) + return head, _verify_pr_head(ctx, pr, head, base_ref=base_ref, remote=remote) + + +def _verify_pr_head(ctx: Context, pr: dict, head: str, *, base_ref: str, remote: str) -> list[str]: + number = int(pr["number"]) + ctx.git.run("fetch", "--quiet", "--no-tags", remote, + f"+refs/pull/{number}/head:refs/kb-gate/pr-{number}", f"+refs/heads/{base_ref}:refs/kb-gate/base") + if ctx.git.rev(f"refs/kb-gate/pr-{number}") != head: + return [f"fetched head does not match PR #{number}'s head {head[:12]} (retry)"] + base_tip = ctx.git.rev("refs/kb-gate/base") + merge_base = ctx.git.merge_base(base_tip, head) + if not touches_knowledge(ctx.git.raw_diff(merge_base, head)): + return [] + try: + final = ctx.git.merge_tree(base_tip, head) + except GateError as exc: + return [str(exc)] + problems = verify_change(ctx, pr=pr, head_sha=head, pre=merge_base, post=head, final=final, + effective_base=base_tip, stage="pull_request") + return problems + tree_problems(ctx.git, base_tip, final) + + +def _pr_for_head(ctx: Context, sha: str) -> dict: + matches = [pr for pr in ctx.github.get_all(f"/repos/{ctx.repository}/pulls?state=open") + if str((pr.get("head") or {}).get("sha")) == sha] + if len(matches) != 1: + raise GateError(f"{len(matches)} open PRs have head {sha[:12]}; the group cannot be attributed") + return ctx.github.get(f"/repos/{ctx.repository}/pulls/{int(matches[0]['number'])}") + + +def verify_merge_group(ctx: Context, head_sha: str, *, base_ref: str = "main", + remote: str = "origin") -> list[str]: + """Every segment of the group, each against its own first parent.""" + ctx.git.run("fetch", "--quiet", "--no-tags", remote, f"+refs/heads/{base_ref}:refs/kb-gate/base") + main_tip = ctx.git.rev("refs/kb-gate/base") + if not ctx.git.is_ancestor(main_tip, head_sha): + return [f"the merge group {head_sha[:12]} is not built on the current {base_ref}"] + chain = ctx.git.first_parent_chain(main_tip, head_sha) + if not chain or chain[-1] != head_sha: + return ["the merge group has no segments"] + problems: list[str] = [] + any_knowledge = False + for commit in chain: + parents = ctx.git.parents(commit) + if len(parents) != 2: + return [f"segment {commit[:12]} is not a two-parent merge commit; the group cannot be attributed"] + first, second = parents + if not touches_knowledge(ctx.git.raw_diff(first, commit)): + continue + any_knowledge = True + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + try: + pr = _pr_for_head(ctx, second) + except GateError as exc: + return [str(exc)] + found = verify_change(ctx, pr=pr, head_sha=second, pre=first, post=commit, final=head_sha, + effective_base=first, stage="merge_group") + problems.extend(f"PR #{pr['number']}: {p}" for p in found) + if any_knowledge: + problems.extend(tree_problems(ctx.git, main_tip, head_sha)) + return problems + + +# -- entry point ---------------------------------------------------------------------- + +def _load_context(args, github) -> Context: + root = Path(args.root) + config_path = Path(__file__).resolve().parent / "config.json" + config = json.loads(config_path.read_text()) if config_path.is_file() else {} + pub = root / ".github" / "kb-gate.pub" + public_key = load_public_key(pub.read_text()) if pub.is_file() else None + codeowners_path = root / ".github" / "CODEOWNERS" + holds_url = str(config.get("holds_url") or "") + return Context( + git=Git(root), github=github, repository=args.repository, public_key=public_key, + holds_loader=lambda: fetch_holds(holds_url), + codeowners=parse_codeowners(codeowners_path.read_text() if codeowners_path.is_file() else ""), + now=time.time(), + ) + + +def main(argv: list[str] | None = None, *, github=None) -> int: + parser = argparse.ArgumentParser(prog="kb-gate") + sub = parser.add_subparsers(dest="stage", required=True) + pr_stage = sub.add_parser("pr") + pr_stage.add_argument("--pr", type=int, required=True) + pr_stage.add_argument("--post-status", action="store_true") + pr_stage.add_argument("--target-url", default="") + group = sub.add_parser("merge-group") + group.add_argument("--head-sha", required=True) + for stage in (pr_stage, group): + stage.add_argument("--repository", required=True) + stage.add_argument("--root", default=".") + stage.add_argument("--base-ref", default="main") + args = parser.parse_args(argv) + github = github or GitHub(args.repository, os.environ.get("GITHUB_TOKEN", ""), + os.environ.get("GITHUB_API_URL", "https://api.github.com")) + ctx = _load_context(args, github) + head = "" + try: + if args.stage == "pr": + head = str(github.get(f"/repos/{args.repository}/pulls/{args.pr}")["head"]["sha"]) + head, problems = verify_pr(ctx, args.pr, base_ref=args.base_ref) + else: + problems = verify_merge_group(ctx, args.head_sha, base_ref=args.base_ref) + except (GateError, SignatureError, VerdictError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems = [f"verifier error: {exc}"] + for problem in problems: + print(f"kb-gate: {problem}") + print("kb-gate: " + ("FAIL" if problems else "PASS")) + if args.stage == "pr" and args.post_status and head: + description = (problems[0] if problems else "knowledge verdict verified")[:139] + github.post(f"/repos/{args.repository}/statuses/{head}", { + "state": "failure" if problems else "success", "context": STATUS_CONTEXT, + "description": description, **({"target_url": args.target_url} if args.target_url else {})}) + return 0 + return 1 if problems else 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/test/test_kb_gate_verifier.py b/test/test_kb_gate_verifier.py new file mode 100644 index 00000000..cc6b9343 --- /dev/null +++ b/test/test_kb_gate_verifier.py @@ -0,0 +1,484 @@ +"""The kb-gate verifier (PR precheck and merge group) and its pinned bundle, offline.""" + +from __future__ import annotations + +import ast +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from infermatrix_copilot.kb_service.gate import changes_between, run_gate, signable_blocks +from infermatrix_copilot.kb_service.models import ModelReply, ModelRole +from infermatrix_copilot.knowledge_service import gate_verifier as gv +from infermatrix_copilot.knowledge_service.ops import KnowledgeOperation as Op, apply_operations +from infermatrix_copilot.knowledge_service.signing import public_key_text, sign +from infermatrix_copilot.knowledge_service.verdict import build_verdict, manifest_from_files + +ROOT = Path(__file__).resolve().parents[1] +REPO = "acme/kb" +PAGE = "repos/demo/core/rules.md" +JUDGE = ModelRole("judge", "codex", "gpt-6-sol", "medium") +KEY = Ed25519PrivateKey.generate() + + +def _rule(rule_id: str, cite: str = "PR #10") -> str: + return (f"## {rule_id} — keep the demo queue bounded\n\n" + "- 触发：修改 demo queue 的容量或背压逻辑时。\n" + "- 强制：队列满时拒绝新请求并返回明确错误，不能无界增长。\n" + f"- 验收：测试覆盖满队列拒绝路径。 ^[{cite}]\n") + + +def _tree() -> dict[str, str]: + return { + "repos/demo/_index.md": "# demo\n", + "repos/demo/core/_index.md": "# core\n\n- [rules](rules.md)\n", + "repos/demo/api/_index.md": "# api\n", + "repos/demo/_routes.yaml": ( + "schema_version: 1\nowners:\n - owner: core\n path: repos/demo/core/rules.md\n" + " signals: [queue]\n scope_prefixes: [demo/core/]\n"), + PAGE: ("---\ntitle: \"Demo core rules\"\ncreated: 2026-09-01\nupdated: 2026-09-01\n" + "type: rule\ntags: [demo]\nsources: [\"PR #10\"]\n---\n\n# Demo core rules\n\n" + + _rule("DEMO-1a")), + } + + +class _Gateway: + def call_json(self, role, *, system, prompt, validate=None): + if '"directory"' in prompt: + data = {"verdict": "consistent", "conflicts": []} + else: + dims = json.loads(prompt.split("<untrusted_data>\n", 1)[1].rsplit("\n</untrusted_data>", 1)[0]) + data = {"dimensions": {d: "yes" for d in dims["dimensions_to_answer"]}, "reasons": {}} + return ModelReply(role, data, json.dumps(data), role.model, {}, 0.1) + + +# -- a git "remote" with main and PR refs, and the verifier's clone ----------------- + +def _git(cwd: Path, *args: str, env: dict | None = None) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True, + env={**os.environ, **(env or {})}).stdout.strip() + + +def _write(repo: Path, files: dict[str, str | None], *, prefix: str = "knowledge/") -> None: + for rel, text in files.items(): + path = repo / (prefix + rel) + if text is None: + path.unlink() + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _commit(repo: Path, message: str) -> str: + _git(repo, "add", "-A") + _git(repo, "-c", "user.name=t", "-c", "user.email=t@e", "commit", "-q", "--allow-empty", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +class World: + """remote repo + verifier clone + a fake GitHub API.""" + + def __init__(self, tmp_path: Path): + self.remote = tmp_path / "remote" + self.remote.mkdir() + _git(self.remote, "init", "-q", "-b", "main") + _write(self.remote, _tree()) + (self.remote / "src").mkdir() + (self.remote / "src" / "app.py").write_text("x = 1\n") + self.main = _commit(self.remote, "base") + self.ws = tmp_path / "ws" + _git(tmp_path, "clone", "-q", str(self.remote), str(self.ws)) + self.prs: dict[int, dict] = {} + self.comments: dict[int, list[dict]] = {} + self.reviews: dict[tuple[int, int], dict] = {} + self.posts: list[tuple[str, dict]] = [] + self.holds = {"global": False, "repos": [], "prs": []} + self.holds_age = 0.0 + self.codeowners = "/.github/ @alice\n" + + # branches ---------------------------------------------------------------- + def branch(self, number: int, files: dict[str, str | None], *, code: dict[str, str] | None = None, + start: str | None = None) -> str: + _git(self.remote, "checkout", "-q", "-B", f"pr-{number}", start or self.main) + _write(self.remote, files) + if code: + _write(self.remote, code, prefix="") + head = _commit(self.remote, f"pr {number}") + _git(self.remote, "update-ref", f"refs/pull/{number}/head", head) + _git(self.remote, "checkout", "-q", "main") + self.prs[number] = {"number": number, "state": "open", "draft": False, "labels": [], + "head": {"sha": head}, "base": {"ref": "main"}} + return head + + def advance_main(self, files: dict[str, str | None], *, code: dict[str, str] | None = None) -> str: + _git(self.remote, "checkout", "-q", "main") + _write(self.remote, files) + if code: + _write(self.remote, code, prefix="") + self.main = _commit(self.remote, "main moves") + return self.main + + def merge_group(self, numbers: list[int]) -> str: + """gh-readonly-queue style: one merge commit per PR on top of main.""" + _git(self.remote, "checkout", "-q", "-B", "queue", self.main) + for number in numbers: + _git(self.remote, "-c", "user.name=q", "-c", "user.email=q@e", "merge", "-q", "--no-ff", + "-m", f"Merge pull request #{number}", self.prs[number]["head"]["sha"]) + head = _git(self.remote, "rev-parse", "HEAD") + _git(self.remote, "checkout", "-q", "main") + _git(self.ws, "fetch", "-q", "origin", f"+{head}:refs/queue/head") + return head + + # verdicts ---------------------------------------------------------------- + def verdict(self, number: int, head_files: dict[str, str], *, key=KEY, source="auto", pr=None, + head_sha=None, issued_at=None, mutate=None, **kw) -> dict: + base = _tree() + head = {**base, **head_files} + touched = set(head_files) + decision = run_gate(base=base, head=head, changes=changes_between(base, head), external_texts={}, + evidence=[], gateway=_Gateway(), judge=JUDGE, release="v1", repo_dir="repos/demo", + retire_ratio=1.0) + assert decision.status == "pass", decision.reasons + verdict = build_verdict( + repository=REPO, pr=pr or number, head_sha=head_sha or self.prs[number]["head"]["sha"], + context_base_sha=kw.pop("context_base_sha", self.main), + manifest=manifest_from_files({k: v for k, v in base.items() if k in touched}, + {k: v for k, v in head.items() if k in touched}), + blocks=[] if source != "auto" else signable_blocks(decision), + consistency=[] if source != "auto" else [ + {"owner_dir": c["owner_dir"], "verdict": c["verdict"], "pages": c["pages"]} + for c in decision.consistency], + release="v1", upstream={}, facts=[], models={}, source=source, + issued_at=time.time() - 5 if issued_at is None else issued_at, **kw) + if mutate: + mutate(verdict) + envelope = sign("kb-gate-verdict", verdict, key) + self.comments.setdefault(number, []).append( + {"body": f"{gv.VERDICT_MARKER}\n```json\n{json.dumps(envelope, ensure_ascii=False)}\n```\n"}) + return verdict + + # the fake GitHub API ----------------------------------------------------- + def get(self, path: str): + path = path.split("?", 1)[0] + parts = path.strip("/").split("/") + if parts[-2:] == ["comments"] or parts[-1] == "comments": + return self.comments.get(int(parts[-2]), []) + if len(parts) == 7 and parts[5] == "reviews": + return self.reviews[(int(parts[4]), int(parts[6]))] + if parts[-1] == "pulls": + return [pr for pr in self.prs.values() if pr["state"] == "open"] + if parts[-2] == "pulls": + return self.prs[int(parts[-1])] + raise AssertionError(path) + + def get_all(self, path: str, *, pages: int = 10): + return self.get(path) + + def post(self, path: str, body: dict): + self.posts.append((path, body)) + return {} + + def context(self) -> gv.Context: + payload = {"issued_at": time.time() - self.holds_age, "sequence": 1, **self.holds} + return gv.Context(git=gv.Git(self.ws), github=self, repository=REPO, public_key=KEY.public_key(), + holds_loader=lambda: sign("kb-holds", payload, KEY), + codeowners=gv.parse_codeowners(self.codeowners), now=time.time()) + + +def _add_rule() -> dict[str, str]: + base = _tree() + return apply_operations(base, [Op("add", PAGE, "DEMO-2a", _rule("DEMO-2a", "PR #11"))], + release="v1", today="2026-09-28").files + + +def _pr(world: World, ctx=None, number: int = 1) -> list[str]: + _head, problems = gv.verify_pr(ctx or world.context(), number) + return problems + + +@pytest.fixture +def world(tmp_path): + return World(tmp_path) + + +# -- PR stage ---------------------------------------------------------------------- + +def test_a_pr_that_does_not_touch_knowledge_passes_without_a_verdict(world): + world.branch(1, {}, code={"src/app.py": "x = 2\n"}) + assert _pr(world) == [] + + +def test_a_signed_auto_verdict_for_this_head_passes(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + assert _pr(world) == [] + + +def test_a_knowledge_pr_without_a_valid_verdict_fails(world): + files = _add_rule() + world.branch(1, files) + assert any("no valid signed verdict" in p for p in _pr(world)) + # replayed from another PR, bound to another head, signed by another key, expired + world.verdict(1, files, pr=2) + world.verdict(1, files, head_sha="f" * 40) + world.verdict(1, files, key=Ed25519PrivateKey.generate()) + world.verdict(1, files, issued_at=time.time() - 73 * 3600) + problems = _pr(world) + assert len(problems) == 1 and "no valid signed verdict" in problems[0] + assert "another repository or pull request" in problems[0] + + +def test_a_verdict_whose_manifest_omits_a_change_fails(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files, mutate=lambda v: v.update(manifest=[])) + assert "the signed patch manifest does not match the change as applied" in _pr(world) + + +def test_a_block_table_that_does_not_cover_the_change_fails(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files, mutate=lambda v: v.update(blocks=v["blocks"][1:])) + assert "the signed block table does not cover exactly the changed blocks" in _pr(world) + + +def test_mixed_code_and_knowledge_and_executable_files_fail_even_with_a_verdict(world): + files = _add_rule() + world.branch(1, files, code={"src/app.py": "x = 3\n"}) + world.verdict(1, files) + problems = _pr(world) + assert "the signed patch manifest does not match the change as applied" in problems + assert any(p.startswith("L1 path_not_whitelisted src/app.py") for p in problems) + + world.branch(2, {}) + _git(world.remote, "checkout", "-q", "pr-2") + page = world.remote / "knowledge" / PAGE + page.write_text(page.read_text() + "\n") + page.chmod(0o755) + head = _commit(world.remote, "exec bit") + _git(world.remote, "update-ref", "refs/pull/2/head", head) + _git(world.remote, "checkout", "-q", "main") + world.prs[2]["head"]["sha"] = head + problems = _pr(world, number=2) + assert any(p.startswith("L1 ") and "100755" in p for p in problems), problems + + +def test_holds_labels_and_a_stale_or_missing_hold_list_fail_closed(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + assert _pr(world) == [] + world.prs[1]["labels"] = [{"name": "kb:hold"}] + assert "PR #1 carries kb:hold" in _pr(world) + world.prs[1]["labels"] = [] + for holds, needle in (({"global": True}, "global pause"), ({"prs": [1]}, "hold list"), + ({"repos": ["demo"]}, "repository demo is paused")): + world.holds = {"global": False, "repos": [], "prs": [], **holds} + assert any(needle in p for p in _pr(world)), holds + world.holds = {"global": False, "repos": [], "prs": []} + world.holds_age = 11 * 60 + assert any("stale" in p for p in _pr(world)) + world.holds_age = 0 + ctx = world.context() + + def unreachable(): + raise gv.GateError("hold list unreachable: timeout") + ctx.holds_loader = unreachable + assert any("unreachable" in p for p in _pr(world, ctx)) + + +def test_a_context_change_since_judging_invalidates_the_verdict(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + world.advance_main({}, code={"src/app.py": "x = 9\n"}) # unrelated: still fine + assert _pr(world) == [] + world.advance_main({"repos/demo/core/_index.md": "# core\n\n- [rules](rules.md)\n\nmore\n"}) + problems = _pr(world) + assert any("context changed since the verdict was judged: repos/demo/core/_index.md" in p for p in problems) + + +def test_the_status_is_posted_on_the_verified_head(world): + files = _add_rule() + head = world.branch(1, files) + world.verdict(1, files) + root = world.ws + (root / ".github").mkdir() + (root / ".github" / "kb-gate.pub").write_text(public_key_text(KEY.public_key()) + "\n") + original = gv.fetch_holds + payload = {"issued_at": time.time(), "sequence": 1, "global": False, "repos": [], "prs": []} + gv.fetch_holds = lambda url: sign("kb-holds", payload, KEY) + try: + assert gv.main(["pr", "--repository", REPO, "--pr", "1", "--root", str(root), "--post-status"], + github=world) == 0 + finally: + gv.fetch_holds = original + assert world.posts == [(f"/repos/{REPO}/statuses/{head}", + {"state": "success", "context": "kb-gate", "description": "knowledge verdict verified"})] + + +# -- human-approved ------------------------------------------------------------------ + +def test_a_human_approved_verdict_needs_a_live_maintainer_approval_on_this_head(world): + files = _add_rule() + head = world.branch(1, files) + world.verdict(1, files, source="human-approved", review_ids=(7,), reviewers=("alice",)) + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + world.reviews[(1, 7)] = {"state": "APPROVED", "commit_id": head, "user": {"login": "alice"}} + assert _pr(world) == [] + world.reviews[(1, 7)]["state"] = "DISMISSED" + assert "review 7 is no longer APPROVED" in _pr(world) + world.reviews[(1, 7)]["state"] = "APPROVED" + world.prs[1]["labels"] = [] + assert "kb:human-approved was removed" in _pr(world) + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + world.codeowners = "/.github/ @bob\n" + assert any("knowledge maintainer" in p for p in _pr(world)) + + +def test_a_human_approval_admits_paths_outside_the_auto_merge_whitelist(world): + tool = {"tools/check.py": "print('ok')\n"} + head = world.branch(1, tool) + manifest = manifest_from_files({}, tool) + world.verdict(1, {}, source="human-approved", review_ids=(7,), reviewers=("alice",), + mutate=lambda v: v.update(manifest=manifest)) + world.reviews[(1, 7)] = {"state": "APPROVED", "commit_id": head, "user": {"login": "alice"}} + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + assert _pr(world) == [] + world.verdict(1, {}, mutate=lambda v: v.update(manifest=manifest, source="auto")) + world.comments[1] = world.comments[1][1:] # only the auto verdict remains + assert any(p.startswith("L1 path_not_whitelisted knowledge/tools/check.py") for p in _pr(world)) + + +def test_a_human_approved_edit_of_a_non_governed_markdown_file_passes(world): + world.advance_main({"skills/demo/SKILL.md": "# demo skill\n"}) + edit = {"skills/demo/SKILL.md": "# demo skill\n\nmore\n"} + head = world.branch(1, edit) + manifest = manifest_from_files({"skills/demo/SKILL.md": "# demo skill\n"}, edit) + world.verdict(1, {}, source="human-approved", review_ids=(7,), reviewers=("alice",), + mutate=lambda v: v.update(manifest=manifest)) + world.reviews[(1, 7)] = {"state": "APPROVED", "commit_id": head, "user": {"login": "alice"}} + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + assert _pr(world) == [] + + +def test_a_hold_on_general_applies_to_general_pages(world): + world.branch(1, {"general/guide.md": "# guide\n"}) + world.holds["repos"] = ["general"] + assert "repository general is paused by the knowledge service" in _pr(world) + + +# -- merge group ------------------------------------------------------------------- + +def test_the_merge_group_verifies_every_knowledge_segment(world): + files = _add_rule() + world.branch(1, files) + world.branch(2, {}, code={"src/app.py": "x = 5\n"}) + world.verdict(1, files) + # knowledge first, code last: the knowledge segment is still verified + assert gv.verify_merge_group(world.context(), world.merge_group([1, 2])) == [] + assert gv.verify_merge_group(world.context(), world.merge_group([2, 1])) == [] + world.comments[1] = [] + problems = gv.verify_merge_group(world.context(), world.merge_group([1, 2])) + assert problems and all(p.startswith("PR #1: ") for p in problems) + + +def test_the_merge_group_rechecks_draft_and_hold_state(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + group = world.merge_group([1]) + world.prs[1]["draft"] = True + assert "PR #1: PR #1 is a draft (paused)" in gv.verify_merge_group(world.context(), group) + world.prs[1]["draft"] = False + world.holds["prs"] = [1] + assert any("hold list" in p for p in gv.verify_merge_group(world.context(), group)) + + +def test_a_merge_group_segment_that_is_not_a_merge_commit_is_rejected(world): + files = _add_rule() + head = world.branch(1, files) + world.verdict(1, files) + _git(world.ws, "fetch", "-q", "origin", f"+{head}:refs/queue/head") # fast-forward "group" + problems = gv.verify_merge_group(world.context(), head) + assert problems == [f"segment {head[:12]} is not a two-parent merge commit; the group cannot be attributed"] + + +def test_a_later_queued_pr_in_the_same_owner_directory_invalidates_the_earlier_verdict(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + other = {"repos/demo/core/_index.md": "# core\n\n- [rules](rules.md)\n\nnote\n"} + world.branch(3, other) + world.verdict(3, other) # judged on main; PR 1 lands in front of it in the group + problems = gv.verify_merge_group(world.context(), world.merge_group([1, 3])) + assert any(p.startswith("PR #3: ") and "context changed" in p for p in problems) + + +def test_a_rule_retired_in_front_of_a_pr_that_cites_it_fails_in_the_landing_tree(world): + base = _tree() + retire = apply_operations(base, [Op("retire", PAGE, "DEMO-1a", reason="upstream-removed", + evidence="PR #13")], release="v1", today="2026-09-28").files + text = _rule("DEMO-3a", "PR #12").replace("队列满时", "与 DEMO-1a 一致，队列满时") + cite = apply_operations(base, [Op("add", "repos/demo/api/rules.md", "DEMO-3a", text, + page_title="Demo api rules")], release="v1", today="2026-09-28").files + world.branch(1, retire) + world.branch(3, cite) + world.verdict(1, retire) + world.verdict(3, cite) + assert _pr(world, number=1) == [] and _pr(world, number=3) == [] # each is fine alone + for order in ([1, 3], [3, 1]): + problems = gv.verify_merge_group(world.context(), world.merge_group(order)) + assert any(p.startswith("PR #1: L1 dangling_reference repos/demo/api/rules.md DEMO-1a") + for p in problems), (order, problems) + + +# -- the pinned bundle ------------------------------------------------------------- + +def test_the_bundle_is_fresh(): + proc = subprocess.run([sys.executable, str(ROOT / "tools" / "build_kb_gate_bundle.py"), "--check"], + capture_output=True, text=True) + assert proc.returncode == 0, proc.stdout + + +def test_the_bundle_imports_only_the_stdlib_its_siblings_and_pinned_deps(): + bundle = ROOT / ".github" / "kb-gate" + siblings = {p.stem for p in bundle.glob("*.py")} + allowed = siblings | {"yaml", "cryptography"} | set(sys.stdlib_module_names) + for path in bundle.glob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + names = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ""] + for name in names: + assert name.split(".")[0] in allowed, f"{path.name} imports {name}" + lock = (bundle / "requirements.lock").read_text() + for pin in ("cryptography==", "cffi==", "pycparser==", "PyYAML=="): + assert pin in lock + assert "--hash=sha256:" in lock + + +def test_the_bundled_verifier_runs_standalone(world, tmp_path): + """Run .github/kb-gate/kb_gate.py as the workflow does, outside the source + tree: a code-only group passes, a knowledge group without the key fails.""" + world.branch(2, {}, code={"src/app.py": "x = 5\n"}) + group = world.merge_group([2]) + script = ROOT / ".github" / "kb-gate" / "kb_gate.py" + env = {"PATH": os.environ["PATH"], "HOME": str(tmp_path), "GITHUB_TOKEN": "none"} + run = lambda head: subprocess.run( # noqa: E731 + [sys.executable, "-E", "-s", str(script), "merge-group", "--repository", REPO, + "--head-sha", head, "--root", str(world.ws)], cwd=tmp_path, env=env, capture_output=True, text=True) + proc = run(group) + assert proc.returncode == 0, proc.stdout + proc.stderr + world.branch(1, _add_rule()) + proc = run(world.merge_group([1])) + assert proc.returncode == 1 and "kb-gate.pub is not committed" in proc.stdout diff --git a/tools/build_kb_gate_bundle.py b/tools/build_kb_gate_bundle.py new file mode 100644 index 00000000..71dfb73a --- /dev/null +++ b/tools/build_kb_gate_bundle.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Generate (or check) the pinned kb-gate verifier bundle in ``.github/kb-gate/``. + +The required check ``kb-gate`` runs ONLY from this bundle, never from ``src/``, +``tools/`` or ``knowledge/tools/``: weakening validation logic in a code PR does +not weaken the gate, and changing the bundle is a ``.github/**`` change that +CODEOWNERS must approve. The bundle is a verbatim copy of the verifier modules +with a generated-file header, plus ``requirements.lock`` (dependencies pinned by +hash) and ``MANIFEST.sha256`` (every bundle file's hash). + + python tools/build_kb_gate_bundle.py # regenerate the module copies + manifest + python tools/build_kb_gate_bundle.py --check # exit 1 when the bundle is stale + python tools/build_kb_gate_bundle.py --lock # re-pin requirements.lock from PyPI (network) + +A stale bundle keeps the gate at its previous strictness; ``--check`` in CI only +reminds that the source moved on. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import urllib.request +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +SOURCE = REPO / "src" / "infermatrix_copilot" / "knowledge_service" +BUNDLE = REPO / ".github" / "kb-gate" +# bundle file -> source module (the verifier imports its siblings by flat name) +MODULES = { + "kb_gate.py": "gate_verifier.py", + "l1.py": "l1.py", + "lifecycle.py": "lifecycle.py", + "ops.py": "ops.py", + "signing.py": "signing.py", + "verdict.py": "verdict.py", +} +# the pinned runtime dependencies: signing needs cryptography (-> cffi -> +# pycparser), l1/lifecycle/ops need PyYAML +PINS = {"cryptography": "50.0.1", "cffi": "2.1.1", "pycparser": "3.0", "PyYAML": "6.0.3"} +STATIC = ("README.md", "config.json", "requirements.lock") +HEADER = "# GENERATED by tools/build_kb_gate_bundle.py from {src} -- do not edit here.\n" + + +def module_text(bundle_name: str) -> str: + src = SOURCE / MODULES[bundle_name] + return HEADER.format(src=src.relative_to(REPO).as_posix()) + src.read_text(encoding="utf-8") + + +def manifest_text(files: dict[str, str]) -> str: + return "".join(f"{hashlib.sha256(files[name].encode()).hexdigest()} {name}\n" for name in sorted(files)) + + +def expected_bundle() -> dict[str, str]: + files = {name: module_text(name) for name in MODULES} + for name in STATIC: + path = BUNDLE / name + if not path.is_file(): + raise SystemExit(f"{path.relative_to(REPO)} is missing") + files[name] = path.read_text(encoding="utf-8") + files["MANIFEST.sha256"] = manifest_text(files) + return files + + +def lock_text() -> str: + lines = ["# GENERATED by tools/build_kb_gate_bundle.py --lock. Install with:", + "# pip install --require-hashes --no-deps -r .github/kb-gate/requirements.lock"] + for name, version in PINS.items(): + with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=30) as resp: + data = json.load(resp) + hashes = sorted({item["digests"]["sha256"] for item in data["urls"]}) + lines.append(f"{name}=={version} \\") + lines.extend(f" --hash=sha256:{digest} \\" for digest in hashes[:-1]) + lines.append(f" --hash=sha256:{hashes[-1]}") + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true") + parser.add_argument("--lock", action="store_true") + args = parser.parse_args(argv) + if args.lock: + (BUNDLE / "requirements.lock").write_text(lock_text(), encoding="utf-8") + expected = expected_bundle() + actual = {p.name: p.read_text(encoding="utf-8") for p in BUNDLE.iterdir() if p.is_file()} \ + if BUNDLE.is_dir() else {} + stale = sorted(name for name in set(expected) | set(actual) if expected.get(name) != actual.get(name)) + if args.check: + for name in stale: + print(f"kb-gate bundle is stale: .github/kb-gate/{name}") + if stale: + print("regenerate with: python tools/build_kb_gate_bundle.py (a .github/** change: CODEOWNERS review)") + return 1 if stale else 0 + BUNDLE.mkdir(parents=True, exist_ok=True) + for name, text in expected.items(): + (BUNDLE / name).write_text(text, encoding="utf-8") + for name in set(actual) - set(expected): + (BUNDLE / name).unlink() + print(f"kb-gate bundle: {len(expected)} files" + (f", updated {', '.join(stale)}" if stale else ", unchanged")) + return 0 + + +if __name__ == "__main__": + sys.exit(main())