From 182a8c15eb2b02ed1cdb40eb4419afc9c3717c31 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 28 Sep 2026 20:46:20 +0800 Subject: [PATCH] Add the kb-gate verifier, its pinned bundle and workflow The repository-side required check for knowledge PRs. - knowledge_service/gate_verifier: verifies a knowledge change from git objects only (the PR is never checked out or executed). A change that does not touch knowledge/ passes. A knowledge change needs a valid Ed25519 verdict bound to this repository, PR and exact head, inside its issue window; a signed manifest equal to the change as applied (base independent); L1 on the change and on the tree that would land. Rules it retires or purges must not be cited anywhere in that landing tree. For auto: a block table equal to the L1 blocks, all passing, and consistency judgements covering the same owner directories whose page hashes match the landing tree. The context set must not have changed between the judged base and the effective base. For human-approved: the bound reviews are still APPROVED on this head, one of them by a knowledge maintainer (CODEOWNERS of .github/kb-gate/), and the label is present; the approval replaces the auto-merge whitelist (knowledge/tools, skills, scripts, mixed PRs) but not the rest of L1. In both cases: no kb:hold, and the signed hold list is fresh and holds neither the PR nor its scope (repository name, or "general" for knowledge/general/). It fails closed without a public key, a hold-list URL, an attributable merge group, or with upstream fact attestations it cannot re-check. - Two stages under one name: the PR precheck (pull_request_target, verdict comments, dispatch) posts the kb-gate status on the head it verified; the merge_group job verifies every first-parent segment (two-parent merge commits, PR looked up by its head) against its own first parent. - tools/build_kb_gate_bundle.py vendors the verifier and its L1/lifecycle/ops/ signing/verdict siblings into .github/kb-gate/, with a hash-pinned requirements.lock and MANIFEST.sha256; the workflow installs with --require-hashes and runs only from there. The CI --check is a reminder and does not block. - .github/CODEOWNERS makes knowledge maintainers own .github/, knowledge/tools, knowledge/skills and knowledge/.claude; governed pages have no code owner. - test.yml also runs on merge_group (a merge-queue prerequisite). Not in this change: the service does not yet issue human-approved verdicts (polling kb:human-approved and checking the approval); until then only auto verdicts exist. Co-Authored-By: Claude Opus 5.5 --- .github/CODEOWNERS | 18 + .github/kb-gate/MANIFEST.sha256 | 9 + .github/kb-gate/README.md | 23 + .github/kb-gate/config.json | 3 + .github/kb-gate/kb_gate.py | 692 ++++++++++++++++++ .github/kb-gate/l1.py | 503 +++++++++++++ .github/kb-gate/lifecycle.py | 424 +++++++++++ .github/kb-gate/ops.py | 376 ++++++++++ .github/kb-gate/requirements.lock | 227 ++++++ .github/kb-gate/signing.py | 118 +++ .github/kb-gate/verdict.py | 149 ++++ .github/workflows/kb-gate.yml | 99 +++ .github/workflows/test.yml | 7 + doc/architecture/SPEC/knowledge_service.md | 23 + doc/features/knowledge-service.md | 25 + .../knowledge_service/gate_verifier.py | 691 +++++++++++++++++ test/test_kb_gate_verifier.py | 484 ++++++++++++ tools/build_kb_gate_bundle.py | 107 +++ 18 files changed, 3978 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/kb-gate/MANIFEST.sha256 create mode 100644 .github/kb-gate/README.md create mode 100644 .github/kb-gate/config.json create mode 100644 .github/kb-gate/kb_gate.py create mode 100644 .github/kb-gate/l1.py create mode 100644 .github/kb-gate/lifecycle.py create mode 100644 .github/kb-gate/ops.py create mode 100644 .github/kb-gate/requirements.lock create mode 100644 .github/kb-gate/signing.py create mode 100644 .github/kb-gate/verdict.py create mode 100644 .github/workflows/kb-gate.yml create mode 100644 src/infermatrix_copilot/knowledge_service/gate_verifier.py create mode 100644 test/test_kb_gate_verifier.py create mode 100644 tools/build_kb_gate_bundle.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..749c9593 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,18 @@ +# Knowledge maintainers own the knowledge gate: its verifier bundle, workflows, +# signing key and this file, plus the knowledge tooling and the non-governed +# knowledge paths. With the ruleset's "require review from Code Owners" and +# "dismiss stale approvals", a change here cannot land without a maintainer +# approving its current head (doc/features/knowledge-service.md). +# +# Governed knowledge pages (knowledge/repos/**, knowledge/general/**) have NO +# code owner on purpose: they merge through the kb-gate signed verdict. A +# human-approved verdict must be approved by an owner of /.github/kb-gate/. +# +# GitHub does not let an author approve their own PR, and knowledge PRs are +# opened as @tzhouam, so at least one more maintainer must be listed below +# before the human-approved path can be used (owner setup step). + +/.github/ @tzhouam +/knowledge/tools/ @tzhouam +/knowledge/skills/ @tzhouam +/knowledge/.claude/ @tzhouam diff --git a/.github/kb-gate/MANIFEST.sha256 b/.github/kb-gate/MANIFEST.sha256 new file mode 100644 index 00000000..88a019f3 --- /dev/null +++ b/.github/kb-gate/MANIFEST.sha256 @@ -0,0 +1,9 @@ +5f6e269fbbb951f9a49c40474a3d71c556ac389f5ebd5545f0ac762f85361723 README.md +77d88b9671927ab95bc6ac9ed57aaa24226423e151843381c2a3fa90cd0cafde config.json +e8157d90270e9f437b364e9ef95bc0603c80c6421fa591953525387cc4f5d4e0 kb_gate.py +5743c4a6943c291073c5344d408eaec28e8bc7440a8ee113365588728ded5e8a l1.py +39f69aa816b83b673443601d38e57040521664c5aca407154ba40a8a35bc61d9 lifecycle.py +b0d0fab0210870f989da80f7ffba3978dbf723f4f087025524d1963c4081a009 ops.py +67f2ebd85e4e886f3facc35a181f575fd6291f1e4a4068f5daca99191609aace requirements.lock +e7fbe6f3d6b02fadfbd86e32e67d588ae698ae2de19ac91aa5db2a250cb93946 signing.py +5c5015e7e9e6314f84e0337a58d9aa1bb6f7ec21ec830e4907dc5c35bbca4a09 verdict.py diff --git a/.github/kb-gate/README.md b/.github/kb-gate/README.md new file mode 100644 index 00000000..cb74bce2 --- /dev/null +++ b/.github/kb-gate/README.md @@ -0,0 +1,23 @@ +# kb-gate verifier bundle + +The required check `kb-gate` runs **only** from this directory. It is generated by +`tools/build_kb_gate_bundle.py` from `src/infermatrix_copilot/knowledge_service/` +(`gate_verifier.py` → `kb_gate.py`, plus `l1`, `lifecycle`, `ops`, `signing`, +`verdict`); do not edit the `.py` files here by hand. The job installs +`requirements.lock` with `--require-hashes --no-deps` and checks `MANIFEST.sha256` +before running anything. + +Changing anything here is a `.github/**` change and needs CODEOWNERS approval, so a +code PR that weakens `src/` or `knowledge/tools/` cannot weaken the gate. + +`config.json`: + +- `holds_url`: the public HTTPS URL of the knowledge service's signed hold list + (`/public/holds.json` on the bot host). Until it is set, every PR that + touches `knowledge/` fails closed. PRs that do not touch `knowledge/` always pass. + +The service's public key is `.github/kb-gate.pub` (`ed25519 `, the output of +`kb keygen`). Without it, every knowledge PR fails closed. + +See `doc/features/knowledge-service.md` for the stages (PR precheck and merge group) +and what each checks. diff --git a/.github/kb-gate/config.json b/.github/kb-gate/config.json new file mode 100644 index 00000000..a497977c --- /dev/null +++ b/.github/kb-gate/config.json @@ -0,0 +1,3 @@ +{ + "holds_url": "" +} diff --git a/.github/kb-gate/kb_gate.py b/.github/kb-gate/kb_gate.py new file mode 100644 index 00000000..417ca858 --- /dev/null +++ b/.github/kb-gate/kb_gate.py @@ -0,0 +1,692 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/gate_verifier.py -- do not edit here. +"""kb-gate: the repository-side verifier for knowledge pull requests. + +It is the required check ``kb-gate`` and runs in two places under one name: + +* PR stage (``pull_request_target`` / ``issue_comment`` / dispatch): verifies + the PR against the current base and posts a ``kb-gate`` commit status on the + PR head. It only decides whether a PR may ENTER the merge queue. +* merge group stage (``merge_group``): verifies every segment of the group on + the exact commit that would land and exits non-zero on any failure. This is + what decides whether anything lands. + +A change that does not touch ``knowledge/`` passes. A knowledge change passes +only when all of these hold for it: + +1. L1: whitelisted paths only, regular files only, no new tree issues, on the + PR's own change AND on the tree that would land; +2. a valid Ed25519-signed verdict comment for this repository, PR and exact + head SHA, inside its issue window (the comment's author is irrelevant); +3. the signed patch manifest equals the change as applied (base independent); +4. ``auto``: the signed block table equals the blocks L1 derives, every block + passed, the consistency judgements cover the same owner directories and + their page hashes match the tree that would land; no page in the context + set changed between the judged base and the effective base; + ``human-approved``: every bound review is still APPROVED on this head, one + is by a knowledge maintainer (a CODEOWNERS owner of ``.github/kb-gate/``), + and ``kb:human-approved`` is set; +5. no ``kb:hold`` label and not held by the service's signed hold list (a + missing, forged or stale hold list fails closed). + +PR content is only ever READ through git objects; nothing from the PR is +executed or installed. This module is vendored into ``.github/kb-gate/`` by +``tools/build_kb_gate_bundle.py`` and runs from there, so it imports nothing +outside the standard library and its vendored siblings. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Iterable, Mapping + +try: + from .l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree + from .lifecycle import LifecycleError, Page + from .ops import all_rule_ids, repo_scope + from .signing import SignatureError, load_public_key, verify + from .verdict import VerdictError, check_binding, manifests_equal +except ImportError: # pragma: no cover - vendored verifier bundle + from l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree # type: ignore[no-redef] + from lifecycle import LifecycleError, Page # type: ignore[no-redef] + from ops import all_rule_ids, repo_scope # type: ignore[no-redef] + from signing import SignatureError, load_public_key, verify # type: ignore[no-redef] + from verdict import VerdictError, check_binding, manifests_equal # type: ignore[no-redef] + +STATUS_CONTEXT = "kb-gate" +VERDICT_MARKER = "" +HOLD_LABEL = "kb:hold" +HUMAN_LABEL = "kb:human-approved" +HOLDS_MAX_AGE = 10 * 60 +MAINTAINER_PATH = ".github/kb-gate/kb_gate.py" +WHITELIST_CODES = ("path_not_whitelisted", "mode_not_regular") +KNOWLEDGE_PREFIX = "knowledge/" +GOVERNED_PREFIXES = ("knowledge/repos/", "knowledge/general/") +GOVERNED_SUFFIXES = (".md", ".yaml") +EXTERNAL_REF_DIRS = ("skills/", "plugins/", "adapters/", "doc/", "playbooks/") +EXTERNAL_SUFFIXES = (".md", ".yaml", ".yml", ".json", ".txt") +_ZERO = "0" * 40 + + +class GateError(RuntimeError): + """The verifier could not establish a fact it needs (fails closed).""" + + +# -- git (objects only; nothing from the PR is ever checked out or executed) --- + +class Git: + def __init__(self, cwd: str | Path): + self.cwd = str(cwd) + + def run(self, *args: str, input: bytes | None = None, ok: tuple[int, ...] = (0,)) -> bytes: + proc = subprocess.run(["git", *args], cwd=self.cwd, input=input, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode not in ok: + raise GateError(f"git {' '.join(args)} failed: {proc.stderr.decode(errors='replace')[:400]}") + return proc.stdout + + def rev(self, name: str) -> str: + return self.run("rev-parse", "--verify", "--quiet", f"{name}^{{commit}}").decode().strip() + + def tree_of(self, name: str) -> str: + return self.run("rev-parse", "--verify", f"{name}^{{tree}}").decode().strip() + + def parents(self, sha: str) -> list[str]: + return self.run("rev-list", "--parents", "-n", "1", sha).decode().split()[1:] + + def is_ancestor(self, older: str, newer: str) -> bool: + proc = subprocess.run(["git", "merge-base", "--is-ancestor", older, newer], cwd=self.cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=False) + if proc.returncode not in (0, 1): + raise GateError(f"cannot relate {older[:12]} and {newer[:12]}: " + f"{proc.stderr.decode(errors='replace')[:200]}") + return proc.returncode == 0 + + def merge_base(self, a: str, b: str) -> str: + return self.run("merge-base", a, b).decode().strip() + + def first_parent_chain(self, base: str, head: str) -> list[str]: + return self.run("rev-list", "--first-parent", "--reverse", f"{base}..{head}").decode().split() + + def merge_tree(self, base: str, head: str) -> str: + """The tree a merge of ``head`` into ``base`` would produce (PR stage).""" + proc = subprocess.run(["git", "merge-tree", "--write-tree", base, head], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode != 0 and b"--write-tree" in proc.stderr: # git < 2.38: no --write-tree + return self._index_merge(base, head) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().split()[0] + + def _index_merge(self, base: str, head: str) -> str: + """Only merges where each path changed on at most one side; anything + else is reported as not merging cleanly (stricter, never looser).""" + import tempfile + + with tempfile.TemporaryDirectory() as scratch: + env = {**os.environ, "GIT_INDEX_FILE": str(Path(scratch) / "index")} + for args in (["read-tree", "-i", "-m", self.merge_base(base, head), base, head], ["write-tree"]): + proc = subprocess.run(["git", *args], cwd=self.cwd, env=env, check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().strip() + + def raw_diff(self, old: str, new: str) -> list[dict]: + """Every changed path as a manifest entry (renames are D + A).""" + out = self.run("diff", "--raw", "-z", "--no-renames", "--no-abbrev", old, new).split(b"\0") + entries: list[dict] = [] + index = 0 + while index < len(out) - 1: + meta = out[index].decode() + path = out[index + 1].decode("utf-8", "replace") + index += 2 + if not meta.startswith(":"): + raise GateError(f"unexpected git diff output: {meta!r}") + old_mode, new_mode, old_blob, new_blob, status = meta[1:].split() + entries.append({ + "path": path, "status": status[0], + "old_blob": "" if old_blob == _ZERO else old_blob, + "new_blob": "" if new_blob == _ZERO else new_blob, + "old_mode": "" if old_mode == "000000" else old_mode, + "new_mode": "" if new_mode == "000000" else new_mode, + }) + return entries + + def changed_names(self, old: str, new: str) -> list[str]: + out = self.run("diff", "--name-only", "-z", "--no-renames", old, new, "--", "knowledge/") + return [name.decode("utf-8", "replace") for name in out.split(b"\0") if name] + + def texts(self, rev: str, prefixes: tuple[str, ...], suffixes: tuple[str, ...]) -> dict[str, str]: + listing = self.run("ls-tree", "-r", "-z", "--full-tree", rev).split(b"\0") + wanted: list[tuple[str, str]] = [] + for entry in listing: + if not entry: + continue + meta, _, raw_path = entry.partition(b"\t") + mode, kind, oid = meta.decode().split() + name = raw_path.decode("utf-8", "replace") + if kind == "blob" and mode == "100644" and name.startswith(prefixes) and name.endswith(suffixes): + wanted.append((name, oid)) + if not wanted: + return {} + out = self.run("cat-file", "--batch", input="".join(f"{oid}\n" for _, oid in wanted).encode()) + texts: dict[str, str] = {} + cursor = 0 + for name, _oid in wanted: + header_end = out.index(b"\n", cursor) + size = int(out[cursor:header_end].split()[2]) + body = out[header_end + 1: header_end + 1 + size] + cursor = header_end + 1 + size + 1 + try: + texts[name] = body.decode("utf-8") + except UnicodeDecodeError: + continue + return texts + + def knowledge_files(self, rev: str) -> dict[str, str]: + files = self.texts(rev, GOVERNED_PREFIXES, GOVERNED_SUFFIXES) + return {name[len(KNOWLEDGE_PREFIX):]: text for name, text in files.items()} + + def external_texts(self, rev: str) -> dict[str, str]: + return self.texts(rev, EXTERNAL_REF_DIRS, EXTERNAL_SUFFIXES) + + def show(self, rev: str, path: str) -> str | None: + proc = subprocess.run(["git", "show", f"{rev}:{path}"], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + return proc.stdout.decode("utf-8", "replace") if proc.returncode == 0 else None + + +# -- GitHub REST (read, plus the PR-stage status) -------------------------------- + +class GitHub: + def __init__(self, repository: str, token: str, api: str = "https://api.github.com"): + self.repository, self._token, self._api = repository, token, api.rstrip("/") + + def _request(self, method: str, path: str, body: dict | None = None) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(self._api + path, data=data, method=method, headers={ + "Accept": "application/vnd.github+json", "Authorization": f"Bearer {self._token}", + "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "kb-gate"}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read() or b"null") + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"GitHub {method} {path} failed: {exc}") from exc + + def get(self, path: str) -> Any: + return self._request("GET", path) + + def get_all(self, path: str, *, pages: int = 10) -> list: + out: list = [] + sep = "&" if "?" in path else "?" + for page in range(1, pages + 1): + batch = self.get(f"{path}{sep}per_page=100&page={page}") + out.extend(batch) + if len(batch) < 100: + break + return out + + def post(self, path: str, body: dict) -> Any: + return self._request("POST", path, body) + + +# -- inputs from the protected checkout ------------------------------------------- + +def fetch_holds(url: str) -> Any: + if not url: + raise GateError("no hold-list URL is configured (.github/kb-gate/config.json)") + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "kb-gate"}), + timeout=20) as response: + return json.loads(response.read()) + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"hold list unreachable: {exc}") from exc + + +def verify_holds(envelope: Any, public_key, *, now: float) -> dict: + holds = verify("kb-holds", envelope, public_key) + if not 0 <= now - float(holds["issued_at"]) <= HOLDS_MAX_AGE: + raise GateError("hold list is stale (the knowledge service may be down)") + return holds + + +def parse_codeowners(text: str | None) -> list[tuple[str, tuple[str, ...]]]: + rules = [] + for line in (text or "").splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + pattern, *owners = line.split() + rules.append((pattern, tuple(owner.lstrip("@").lower() for owner in owners))) + return rules + + +def _codeowner_match(pattern: str, path: str) -> bool: + anchored = pattern.startswith("/") + pattern = pattern.lstrip("/") + if pattern.endswith("/"): + pattern += "**" + candidates = [path] if anchored or "/" in pattern.rstrip("*") else \ + [path] + [path.split("/", i)[-1] for i in range(1, path.count("/") + 1)] + for candidate in candidates: + if fnmatch.fnmatchcase(candidate, pattern) or fnmatch.fnmatchcase(candidate, pattern + "/**") \ + or (pattern.endswith("/**") and candidate.startswith(pattern[:-2])): + return True + return False + + +def code_owners(rules: list[tuple[str, tuple[str, ...]]], path: str) -> tuple[str, ...]: + """Owners of ``path``: the LAST matching line wins. Team entries never + match a person here (their membership is not readable with the job token), + so a team-only path cannot be human-approved through this verifier.""" + owners: tuple[str, ...] = () + for pattern, names in rules: + if _codeowner_match(pattern, path): + owners = names + return tuple(name for name in owners if "/" not in name) + + +# -- the verification of one change ------------------------------------------------ + +@dataclass +class Context: + git: Git + github: Any + repository: str + public_key: Any # None when .github/kb-gate.pub is missing + holds_loader: Callable[[], Any] + codeowners: list[tuple[str, tuple[str, ...]]] + now: float + _holds: dict | None = None + + def holds(self) -> dict: + if self._holds is None: + self._holds = verify_holds(self.holds_loader(), self.public_key, now=self.now) + return self._holds + + +def governed(path: str) -> bool: + """A page the knowledge lifecycle governs (what ``knowledge_files`` reads).""" + return path.startswith(GOVERNED_PREFIXES) and path.endswith(GOVERNED_SUFFIXES) + + +def touches_knowledge(entries: Iterable[Mapping[str, str]]) -> bool: + return any(entry["path"].startswith(KNOWLEDGE_PREFIX) for entry in entries) + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def verdict_envelopes(comments: Iterable[Mapping[str, Any]]) -> list[Any]: + out = [] + for comment in comments: + body = str(comment.get("body") or "") + if VERDICT_MARKER not in body: + continue + rest = body.split(VERDICT_MARKER, 1)[1] + start = rest.find("```json") + if start < 0: + continue + rest = rest[start + len("```json"):] + end = rest.find("```") + try: + out.append(json.loads(rest[:end] if end >= 0 else rest)) + except ValueError: + continue + return out + + +def find_verdict(ctx: Context, pr: int, head_sha: str) -> tuple[dict | None, list[str]]: + """The newest valid verdict for exactly this PR head, and why others failed.""" + comments = ctx.github.get_all(f"/repos/{ctx.repository}/issues/{pr}/comments") + valid, rejected = [], [] + for envelope in verdict_envelopes(comments): + try: + verdict = verify("kb-gate-verdict", envelope, ctx.public_key) + check_binding(verdict, repository=ctx.repository, pr=pr, head_sha=head_sha, now=ctx.now) + valid.append(verdict) + except (SignatureError, VerdictError, KeyError, TypeError, ValueError) as exc: + rejected.append(str(exc)) + if not valid: + return None, rejected + return max(valid, key=lambda v: float(v["issued_at"])), rejected + + +def _owner_dir(path: str) -> str: + return str(PurePosixPath(path).parent) + + +def _scope_root(path: str) -> str: + parts = PurePosixPath(path).parts + return "/".join(parts[:2]) if parts[:1] == ("repos",) else parts[0] + + +def context_paths(touched: set[str], owner_dirs: set[str], post: Mapping[str, str], + base: Mapping[str, str]) -> Callable[[str], bool]: + """The pages whose change since the judged base invalidates L2: the touched + pages, every page in their owner directories, the scope's _routes.yaml and + _index.md files (scope root and owner directories), and pages holding rules + the change supersedes.""" + roots = {_scope_root(path) for path in touched} + dirs = owner_dirs | {_owner_dir(path) for path in touched} + superseded: set[str] = set() + for path in touched: + text = post.get(path) + if not text or not path.endswith(".md"): + continue + try: + sections = Page.parse(text).rules() + except (LifecycleError, ValueError): + continue + for section in sections: + try: + target = section.footer.supersedes + except LifecycleError: + continue + for rule_id in [t.strip() for t in target.split(",") if t.strip()]: + superseded.update(p for p, t in base.items() + if p.endswith(".md") and rule_id in all_rule_ids({p: t})) + + def relevant(path: str) -> bool: + name = PurePosixPath(path).name + parent = _owner_dir(path) + return (path in touched or path in superseded or parent in dirs + or (name in ("_routes.yaml", "_index.md") and parent in roots | dirs)) + return relevant + + +def verify_change(ctx: Context, *, pr: dict, head_sha: str, pre: str, post: str, final: str, + effective_base: str, stage: str) -> list[str]: + """Problems with one PR's knowledge change (empty when it passes). + + ``pre``/``post`` are the commits (or trees) around this PR's change as it + is applied; ``final`` is the tree that would land; ``effective_base`` is + the state the change is applied on (current main, or the first parent of + the merge-group segment).""" + number = int(pr["number"]) + entries = ctx.git.raw_diff(pre, post) + if not touches_knowledge(entries): + return [] + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + problems: list[str] = [] + labels = {str(label.get("name")) for label in pr.get("labels") or []} + if pr.get("state") != "open": + problems.append(f"PR #{number} is not open") + if str((pr.get("head") or {}).get("sha")) != head_sha: + problems.append(f"PR #{number} head moved from {head_sha[:12]}") + if stage == "merge_group" and pr.get("draft"): + problems.append(f"PR #{number} is a draft (paused)") + if HOLD_LABEL in labels: + problems.append(f"PR #{number} carries {HOLD_LABEL}") + + pre_files, post_files = ctx.git.knowledge_files(pre), ctx.git.knowledge_files(post) + final_files = ctx.git.knowledge_files(final) + changes = [Change(e["path"], e["status"] if e["status"] in "AMD" else "M", e["old_mode"], e["new_mode"]) + for e in entries] + for e in entries: + if e["status"] not in ("A", "M", "D"): + problems.append(f"{e['path']}: change type {e['status']} is not allowed") + touched = {e["path"][len(KNOWLEDGE_PREFIX):] for e in entries if e["path"].startswith(KNOWLEDGE_PREFIX)} + + try: + holds = ctx.holds() + if holds.get("global"): + problems.append("the knowledge service holds every repository (global pause)") + if number in {int(p) for p in holds.get("prs") or []}: + problems.append(f"PR #{number} is on the service's hold list") + held = set(holds.get("repos") or []) + # a scope is its repository name, or "general" for knowledge/general/ + scopes = {repo_scope(p).split("/")[-1] for p in touched if p.startswith(("repos/", "general/"))} + for scope in sorted(scopes & held): + problems.append(f"repository {scope} is paused by the knowledge service") + except (GateError, SignatureError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems.append(f"hold list: {exc}") + + verdict, rejected = find_verdict(ctx, number, head_sha) + if verdict is None: + detail = f" (rejected: {'; '.join(rejected[:3])})" if rejected else "" + problems.append(f"no valid signed verdict for PR #{number} at {head_sha[:12]}{detail}") + # still report L1, so the author sees every structural problem at once + result = check_changeset(pre_files, post_files, changes, external_texts={}) + return problems + [f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues] + + if not manifests_equal(verdict.get("manifest") or [], entries): + problems.append("the signed patch manifest does not match the change as applied") + if verdict.get("facts"): + problems.append("the verdict carries upstream fact attestations this verifier cannot re-check") + + human = verdict["source"] == "human-approved" + # a maintainer's approval replaces the auto-merge whitelist (knowledge/tools, + # skills, scripts, mixed PRs) and L2, never the rest of L1: non-governed + # paths are not pages, so the page lifecycle checks run on governed ones only + l1_changes = [c for c in changes if governed(c.path)] if human else changes + result = check_changeset(pre_files, post_files, l1_changes, + external_texts=ctx.git.external_texts(final), + release=str(verdict.get("release") or "")) + waived = WHITELIST_CODES if human else () + problems.extend(f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues if i.code not in waived) + # what this change retires or purges must not be cited anywhere in the tree + # that would land (a PR queued behind it, or main, may have added a citation) + gone = set(result.retired) | set(result.purged) + if gone: + try: + problems.extend(f"L1 {i.code} {i.path} {i.detail} (in the tree that would land)" + for i in dangling_refs(final_files, gone)) + except (LifecycleError, ValueError) as exc: + problems.append(f"L1 the tree that would land does not parse: {exc}") + + if verdict["source"] == "auto": + if result.external_refs: + problems.append("retired rules are still cited outside knowledge/ (needs a companion PR)") + signed_blocks = verdict.get("blocks") or [] + if {b.block_id for b in result.blocks} != {b.get("block_id") for b in signed_blocks}: + problems.append("the signed block table does not cover exactly the changed blocks") + if any(b.get("verdict") != "pass" for b in signed_blocks): + problems.append("a signed block did not pass") + owner_dirs = {_owner_dir(b.path) for b in result.blocks} + consistency = verdict.get("consistency") or [] + if {c.get("owner_dir") for c in consistency} != owner_dirs: + problems.append("the consistency judgement does not cover exactly the changed owner directories") + for item in consistency: + directory = item.get("owner_dir") + actual = {p: _sha256(t) for p, t in final_files.items() + if _owner_dir(p) == directory and p.endswith(".md")} + if item.get("verdict") != "consistent": + problems.append(f"{directory}: consistency was not judged consistent") + if item.get("pages") != actual: + problems.append(f"{directory}: pages changed since the consistency judgement") + base_sha = str(verdict["context_base_sha"]) + try: + if not ctx.git.is_ancestor(base_sha, effective_base): + problems.append("the judged context base is not an ancestor of the effective base") + else: + relevant = context_paths(touched, owner_dirs, post_files, pre_files) + moved = [n[len(KNOWLEDGE_PREFIX):] for n in ctx.git.changed_names(base_sha, effective_base)] + stale = sorted(p for p in moved if relevant(p)) + if stale: + problems.append("context changed since the verdict was judged: " + ", ".join(stale[:5])) + except GateError as exc: + problems.append(f"context base: {exc}") + else: + problems.extend(_human_approval_problems(ctx, pr, verdict, head_sha, labels)) + return problems + + +def _human_approval_problems(ctx: Context, pr: dict, verdict: dict, head_sha: str, + labels: set[str]) -> list[str]: + problems = [] + if HUMAN_LABEL not in labels: + problems.append(f"{HUMAN_LABEL} was removed") + number = int(pr["number"]) + reviewers = {str(r).lower() for r in verdict.get("reviewers") or []} + approvers = set() + for review_id in verdict.get("review_ids") or []: + review = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}/reviews/{int(review_id)}") + login = str((review.get("user") or {}).get("login") or "").lower() + if review.get("state") != "APPROVED": + problems.append(f"review {review_id} is no longer APPROVED") + elif review.get("commit_id") != head_sha: + problems.append(f"review {review_id} approved another head") + elif login not in reviewers: + problems.append(f"review {review_id} is by someone the verdict does not name") + else: + approvers.add(login) + # knowledge maintainers = the CODEOWNERS of the gate itself. Governed pages + # deliberately have no code owners (that would force a review on every auto + # PR); paths outside the whitelist get GitHub's native code-owner review. + maintainers = set(code_owners(ctx.codeowners, MAINTAINER_PATH)) + if not approvers & maintainers: + problems.append("no valid approving review by a knowledge maintainer (CODEOWNERS of .github/kb-gate/)") + return problems + + +def tree_problems(git: Git, base: str, final: str) -> list[str]: + """L1 on the tree that would land: no tree-level issue that base lacks.""" + before = {(i.code, i.detail) for i in check_tree(git.knowledge_files(base))} + return [f"L1 {i.code} {i.path} {i.detail}".rstrip() + for i in check_tree(git.knowledge_files(final)) if (i.code, i.detail) not in before] + + +# -- the two stages ----------------------------------------------------------------- + +def verify_pr(ctx: Context, number: int, *, base_ref: str = "main", + remote: str = "origin") -> tuple[str, list[str]]: + """(the head SHA verified, its problems). The status goes on THAT head, so + a push during verification can only leave the new head without a status.""" + pr = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}") + head = str(pr["head"]["sha"]) + return head, _verify_pr_head(ctx, pr, head, base_ref=base_ref, remote=remote) + + +def _verify_pr_head(ctx: Context, pr: dict, head: str, *, base_ref: str, remote: str) -> list[str]: + number = int(pr["number"]) + ctx.git.run("fetch", "--quiet", "--no-tags", remote, + f"+refs/pull/{number}/head:refs/kb-gate/pr-{number}", f"+refs/heads/{base_ref}:refs/kb-gate/base") + if ctx.git.rev(f"refs/kb-gate/pr-{number}") != head: + return [f"fetched head does not match PR #{number}'s head {head[:12]} (retry)"] + base_tip = ctx.git.rev("refs/kb-gate/base") + merge_base = ctx.git.merge_base(base_tip, head) + if not touches_knowledge(ctx.git.raw_diff(merge_base, head)): + return [] + try: + final = ctx.git.merge_tree(base_tip, head) + except GateError as exc: + return [str(exc)] + problems = verify_change(ctx, pr=pr, head_sha=head, pre=merge_base, post=head, final=final, + effective_base=base_tip, stage="pull_request") + return problems + tree_problems(ctx.git, base_tip, final) + + +def _pr_for_head(ctx: Context, sha: str) -> dict: + matches = [pr for pr in ctx.github.get_all(f"/repos/{ctx.repository}/pulls?state=open") + if str((pr.get("head") or {}).get("sha")) == sha] + if len(matches) != 1: + raise GateError(f"{len(matches)} open PRs have head {sha[:12]}; the group cannot be attributed") + return ctx.github.get(f"/repos/{ctx.repository}/pulls/{int(matches[0]['number'])}") + + +def verify_merge_group(ctx: Context, head_sha: str, *, base_ref: str = "main", + remote: str = "origin") -> list[str]: + """Every segment of the group, each against its own first parent.""" + ctx.git.run("fetch", "--quiet", "--no-tags", remote, f"+refs/heads/{base_ref}:refs/kb-gate/base") + main_tip = ctx.git.rev("refs/kb-gate/base") + if not ctx.git.is_ancestor(main_tip, head_sha): + return [f"the merge group {head_sha[:12]} is not built on the current {base_ref}"] + chain = ctx.git.first_parent_chain(main_tip, head_sha) + if not chain or chain[-1] != head_sha: + return ["the merge group has no segments"] + problems: list[str] = [] + any_knowledge = False + for commit in chain: + parents = ctx.git.parents(commit) + if len(parents) != 2: + return [f"segment {commit[:12]} is not a two-parent merge commit; the group cannot be attributed"] + first, second = parents + if not touches_knowledge(ctx.git.raw_diff(first, commit)): + continue + any_knowledge = True + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + try: + pr = _pr_for_head(ctx, second) + except GateError as exc: + return [str(exc)] + found = verify_change(ctx, pr=pr, head_sha=second, pre=first, post=commit, final=head_sha, + effective_base=first, stage="merge_group") + problems.extend(f"PR #{pr['number']}: {p}" for p in found) + if any_knowledge: + problems.extend(tree_problems(ctx.git, main_tip, head_sha)) + return problems + + +# -- entry point ---------------------------------------------------------------------- + +def _load_context(args, github) -> Context: + root = Path(args.root) + config_path = Path(__file__).resolve().parent / "config.json" + config = json.loads(config_path.read_text()) if config_path.is_file() else {} + pub = root / ".github" / "kb-gate.pub" + public_key = load_public_key(pub.read_text()) if pub.is_file() else None + codeowners_path = root / ".github" / "CODEOWNERS" + holds_url = str(config.get("holds_url") or "") + return Context( + git=Git(root), github=github, repository=args.repository, public_key=public_key, + holds_loader=lambda: fetch_holds(holds_url), + codeowners=parse_codeowners(codeowners_path.read_text() if codeowners_path.is_file() else ""), + now=time.time(), + ) + + +def main(argv: list[str] | None = None, *, github=None) -> int: + parser = argparse.ArgumentParser(prog="kb-gate") + sub = parser.add_subparsers(dest="stage", required=True) + pr_stage = sub.add_parser("pr") + pr_stage.add_argument("--pr", type=int, required=True) + pr_stage.add_argument("--post-status", action="store_true") + pr_stage.add_argument("--target-url", default="") + group = sub.add_parser("merge-group") + group.add_argument("--head-sha", required=True) + for stage in (pr_stage, group): + stage.add_argument("--repository", required=True) + stage.add_argument("--root", default=".") + stage.add_argument("--base-ref", default="main") + args = parser.parse_args(argv) + github = github or GitHub(args.repository, os.environ.get("GITHUB_TOKEN", ""), + os.environ.get("GITHUB_API_URL", "https://api.github.com")) + ctx = _load_context(args, github) + head = "" + try: + if args.stage == "pr": + head = str(github.get(f"/repos/{args.repository}/pulls/{args.pr}")["head"]["sha"]) + head, problems = verify_pr(ctx, args.pr, base_ref=args.base_ref) + else: + problems = verify_merge_group(ctx, args.head_sha, base_ref=args.base_ref) + except (GateError, SignatureError, VerdictError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems = [f"verifier error: {exc}"] + for problem in problems: + print(f"kb-gate: {problem}") + print("kb-gate: " + ("FAIL" if problems else "PASS")) + if args.stage == "pr" and args.post_status and head: + description = (problems[0] if problems else "knowledge verdict verified")[:139] + github.post(f"/repos/{args.repository}/statuses/{head}", { + "state": "failure" if problems else "success", "context": STATUS_CONTEXT, + "description": description, **({"target_url": args.target_url} if args.target_url else {})}) + return 0 + return 1 if problems else 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/.github/kb-gate/l1.py b/.github/kb-gate/l1.py new file mode 100644 index 00000000..fd75d6d6 --- /dev/null +++ b/.github/kb-gate/l1.py @@ -0,0 +1,503 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/l1.py -- do not edit here. +"""L1: the deterministic half of the knowledge quality gate. + +Two entry points, both pure functions over ``{knowledge-relative path: text}`` +mappings so the service, the gate and the vendored verifier agree exactly: + +* ``check_tree(files)`` — structural issues in one tree (footers, supersede + links, tombstones, ``_routes.yaml``). +* ``check_changeset(base, head, changes)`` — what one change set may do: only + whitelisted files and modes, no new duplicate or reused rule IDs, no dangling + references to rules it retired or purged, no NEW tree issues, and a + classification of every change into blocks: + + ``rule`` a rule section added / edited / retired / superseded / purged. + Needs a per-block L2 verdict (keyed by ``block_id``). + ``prose`` any other wording change (page head, non-rule sections, + guides, ``_routes.yaml``). Needs an L2 prose verdict. + ``mechanical`` ``updated:``/``sources:`` frontmatter, index lines for new + pages, tombstones. Recomputed here; a mismatch is an issue, + never a block a verdict could cover. + +Standard library + PyYAML + ``lifecycle``/``ops`` only (vendored by kb-gate). +""" + +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Iterable, Mapping + +import yaml + +try: + from .lifecycle import ( + FRONTMATTER, LifecycleError, Page, Section, expected_sources, + ) + from .ops import ( + INDEX_NAME, TOMBSTONES_NAME, all_rule_ids, all_tombstoned_ids, + load_tombstones, page_over_capacity, render_tombstones, + ) +except ImportError: # pragma: no cover - vendored verifier bundle + from lifecycle import ( # type: ignore[no-redef] + FRONTMATTER, LifecycleError, Page, Section, expected_sources, + ) + from ops import ( # type: ignore[no-redef] + INDEX_NAME, TOMBSTONES_NAME, all_rule_ids, all_tombstoned_ids, + load_tombstones, page_over_capacity, render_tombstones, + ) + +ROUTES_NAME = "_routes.yaml" +KNOWLEDGE_PREFIX = "knowledge/" +_WHITELIST = ( + re.compile(r"knowledge/(?:repos/[^/]+|general)/(?:[^/]+/)*[^/]+\.md"), + re.compile(r"knowledge/repos/[^/]+/_routes\.yaml"), + re.compile(r"knowledge/(?:repos/[^/]+|general)/_tombstones\.yaml"), +) +_DENY = re.compile(r"knowledge/(?:skills|tools|\.claude)/|knowledge/[^/]+$") +_REGULAR_MODE = "100644" +_INDEX_LINE = re.compile(r"- \[[^\]\n]+\]\((?P[^)/\s]+\.md)\)") + + +@dataclass(frozen=True) +class Issue: + code: str + path: str + detail: str + + def to_dict(self) -> dict: + return {"code": self.code, "path": self.path, "detail": self.detail} + + +@dataclass(frozen=True) +class Change: + """One changed path as git reports it (repository-relative).""" + + path: str + status: str # A, M, D (renames are reported as D + A) + old_mode: str = "" + new_mode: str = "" + + +@dataclass(frozen=True) +class Block: + kind: str # rule | prose + path: str # knowledge-relative + rule_id: str # "" for prose + op: str # add | edit | retire | supersede | purge | prose + sha256: str # of the text the verdict judges + + @property + def block_id(self) -> str: + identity = json.dumps([self.kind, self.path, self.rule_id, self.op, self.sha256], + separators=(",", ":")) + return hashlib.sha256(identity.encode("utf-8")).hexdigest() + + def to_dict(self) -> dict: + return {"block_id": self.block_id, "kind": self.kind, "path": self.path, + "rule_id": self.rule_id, "op": self.op, "sha256": self.sha256} + + +@dataclass(frozen=True) +class ChangesetResult: + issues: tuple[Issue, ...] + blocks: tuple[Block, ...] + retired: tuple[str, ...] + purged: tuple[str, ...] + external_refs: tuple[tuple[str, str], ...] = () # (rule_id, repo path) + + @property + def ok(self) -> bool: + return not self.issues + + +def _sha(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _norm(text: str) -> str: + """Section text without trailing blank lines: appending a section adds a + separator to the previous one, which is layout, not a change of words.""" + return text.rstrip("\n") + "\n" + + +# -- tree-level ---------------------------------------------------------------- + +def _rule_pages(files: Mapping[str, str]) -> dict[str, Page]: + pages: dict[str, Page] = {} + for path, text in files.items(): + if path.endswith(".md"): + page = Page.parse(text) + if page.rules(): + pages[path] = page + return pages + + +def check_tree(files: Mapping[str, str]) -> list[Issue]: + issues: list[Issue] = [] + footers: dict[str, tuple[str, object]] = {} + for path, page in _rule_pages(files).items(): + for section in page.rules(): + try: + footer = section.footer + except LifecycleError as exc: + issues.append(Issue("footer_invalid", path, f"{section.rule_id}: {exc}")) + continue + footers.setdefault(section.rule_id, (path, footer)) + for rule_id, (path, footer) in footers.items(): + if footer.superseded_by: + target = footers.get(footer.superseded_by) + if target is None or target[1].supersedes != rule_id: + issues.append(Issue("supersede_link_broken", path, + f"{rule_id} -> {footer.superseded_by} has no matching supersedes")) + if footer.supersedes: + source = footers.get(footer.supersedes) + if source is not None and source[1].superseded_by != rule_id: + issues.append(Issue("supersede_link_broken", path, + f"{rule_id} supersedes {footer.supersedes}, which does not name it")) + try: + tombstoned = all_tombstoned_ids(files) + except (LifecycleError, yaml.YAMLError) as exc: + issues.append(Issue("tombstones_invalid", "", str(exc))) + tombstoned = set() + for rule_id in sorted(tombstoned & set(all_rule_ids(files))): + issues.append(Issue("tombstoned_id_reused", "", rule_id)) + for path, text in files.items(): + if PurePosixPath(path).name == ROUTES_NAME: + issues.extend(_check_routes(path, text, files)) + return issues + + +def _check_routes(path: str, text: str, files: Mapping[str, str]) -> list[Issue]: + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + return [Issue("routes_invalid", path, f"not YAML: {exc}")] + if not isinstance(data, dict) or data.get("schema_version") != 1: + return [Issue("routes_invalid", path, "schema_version must be 1")] + issues = [] + seen: set[str] = set() + for index, owner in enumerate(data.get("owners") or []): + if not isinstance(owner, dict) or not owner.get("owner") or not owner.get("path"): + issues.append(Issue("routes_invalid", path, f"owners[{index}] needs owner and path")) + continue + if owner["owner"] in seen: + issues.append(Issue("routes_invalid", path, f"duplicate owner {owner['owner']}")) + seen.add(owner["owner"]) + if str(owner["path"]) not in files: + issues.append(Issue("routes_page_missing", path, str(owner["path"]))) + for key in ("signals", "scope_prefixes"): + if not isinstance(owner.get(key) or [], list): + issues.append(Issue("routes_invalid", path, f"{owner['owner']}.{key} must be a list")) + models = data.get("models") + if models is not None and (not isinstance(models, dict) or not models.get("dir") or not models.get("page")): + issues.append(Issue("routes_invalid", path, "models needs dir and page")) + return issues + + +# -- change-set level ---------------------------------------------------------- + +def _whitelist_issues(changes: Iterable[Change]) -> list[Issue]: + issues = [] + for change in changes: + path = change.path + if not path.startswith(KNOWLEDGE_PREFIX) or _DENY.match(path) \ + or not any(p.fullmatch(path) for p in _WHITELIST): + issues.append(Issue("path_not_whitelisted", path, "only knowledge rule/guide pages, _routes.yaml and _tombstones.yaml auto-merge")) + continue + if change.status != "D" and change.new_mode != _REGULAR_MODE: + issues.append(Issue("mode_not_regular", path, f"mode {change.new_mode or '?'} (need 100644)")) + return issues + + +def _text_issues(path: str, text: str | None) -> list[Issue]: + if text is None: + return [] + if "\x00" in text or "�" in text: + return [Issue("not_utf8_text", path, "binary or non-UTF-8 content")] + return [] + + +def _frontmatter_mechanical(path: str, base: Page, head: Page) -> list[Issue]: + """Only ``updated:`` and ``sources:`` may change, and sources exactly as recomputed.""" + base_data, head_data = base.frontmatter_data(), head.frontmatter_data() + issues = [] + for key in sorted(set(base_data) | set(head_data)): + if key in ("updated", "sources"): + continue + if base_data.get(key) != head_data.get(key): + issues.append(Issue("frontmatter_changed", path, f"{key} may not change")) + if str(head_data.get("updated", "")) < str(base_data.get("updated", "")): + issues.append(Issue("frontmatter_changed", path, "updated went backwards")) + want = expected_sources(base.sources(), head) + if head.sources() != want: + issues.append(Issue("sources_mismatch", path, f"expected {want}, got {head.sources()}")) + return issues + + +def _new_page_mechanical(path: str, head: Page) -> list[Issue]: + data = head.frontmatter_data() + issues = [] + if data.get("type") != "rule": + issues.append(Issue("new_page_invalid", path, "a new page must be type: rule")) + if not str(data.get("title") or "").strip(): + issues.append(Issue("new_page_invalid", path, "a new page needs a title")) + want = expected_sources([], head) + if head.sources() != want: + issues.append(Issue("sources_mismatch", path, f"expected {want}, got {head.sources()}")) + return issues + + +def _classify_page(path: str, base_text: str | None, head_text: str | None, + blocks: list[Block], issues: list[Issue], + retired: list[str], purged: list[tuple[str, str]], + purge_retired_at: dict[str, str]) -> None: + base = Page.parse(base_text) if base_text is not None else None + head = Page.parse(head_text) if head_text is not None else None + if head is None: + issues.append(Issue("page_deleted", path, "pages are never deleted; purge rules instead")) + return + base_is_rule = base is not None and ( + base.frontmatter_data().get("type") == "rule" or bool(base.rules())) + is_rule_page = base_is_rule or head.frontmatter_data().get("type") == "rule" + if base_is_rule and head.frontmatter_data().get("type") != base.frontmatter_data().get("type"): + # a page may not leave the lifecycle by changing its type + issues.append(Issue("page_type_changed", path, "a rule page keeps its type")) + if not is_rule_page: + if base is None or base.render() != head.render(): + blocks.append(Block("prose", path, "", "prose", _sha(head.render()))) + return + if base is None: + issues.extend(_new_page_mechanical(path, head)) + base_rules: dict[str, Section] = {} + base_other: list[str] = [] + base_head = "" + else: + issues.extend(_frontmatter_mechanical(path, base, head)) + base_rules = {s.rule_id: s for s in base.rules()} + base_other = [_norm(s.text) for s in base.sections if not s.is_rule] + base_head = base.head + head_rules = {s.rule_id: s for s in head.rules()} + head_other = [_norm(s.text) for s in head.sections if not s.is_rule] + if (base is None and head.head.strip() != f"# {head.frontmatter_data().get('title', '')}") \ + or (base is not None and head.head != base_head) or head_other != base_other: + blocks.append(Block("prose", path, "", "prose", + _sha(head.head + "".join(head_other)))) + for rule_id, section in head_rules.items(): + old = base_rules.get(rule_id) + try: + footer = section.footer + old_footer = old.footer if old is not None else None + except LifecycleError as exc: + issues.append(Issue("footer_invalid", path, f"{rule_id}: {exc}")) + continue + if old is None: + if footer.status != "active": + issues.append(Issue("rule_added_retired", path, rule_id)) + blocks.append(Block("rule", path, rule_id, "add", _sha(_norm(section.text)))) + continue + if _norm(old.text) == _norm(section.text): + continue + if old_footer.status == "retired": + issues.append(Issue("retired_rule_changed", path, f"{rule_id} is retired and frozen")) + continue + if old_footer.protected and not footer.protected: + issues.append(Issue("protection_removed", path, rule_id)) + if footer.status == "retired": + if section.body_without_footer != old.body_without_footer: + issues.append(Issue("retire_changed_text", path, f"{rule_id}: retiring may not reword")) + if old_footer.protected or footer.protected: + issues.append(Issue("protected_rule_retired", path, rule_id)) + op = "supersede" if footer.reason == "superseded" else "retire" + retired.extend(dict.fromkeys((rule_id, *section.nested_rule_ids))) + blocks.append(Block("rule", path, rule_id, op, _sha(_norm(section.text)))) + else: + if footer != old_footer: + issues.append(Issue("footer_changed", path, f"{rule_id}: an edit keeps its footer")) + if old_footer.protected or footer.protected: + issues.append(Issue("protected_rule_edited", path, rule_id)) + if set(section.nested_rule_ids) != set(old.nested_rule_ids): + # a nested rule leaves only by retiring and purging its parent + issues.append(Issue("nested_rule_ids_changed", path, + f"{rule_id}: {sorted(set(old.nested_rule_ids) ^ set(section.nested_rule_ids))}")) + blocks.append(Block("rule", path, rule_id, "edit", _sha(_norm(section.text)))) + for rule_id, old in base_rules.items(): + if rule_id in head_rules: + continue + if old.footer.status != "retired": + issues.append(Issue("active_rule_removed", path, f"{rule_id}: retire before purging")) + continue + if old.footer.protected: + issues.append(Issue("protected_rule_purged", path, rule_id)) + purged.extend((rid, path) for rid in dict.fromkeys((rule_id, *old.nested_rule_ids))) + purge_retired_at[rule_id] = old.footer.retired_at + blocks.append(Block("rule", path, rule_id, "purge", _sha(_norm(old.text)))) + + +def _index_mechanical(path: str, base_text: str | None, head_text: str | None, + new_pages: set[str], issues: list[Issue], blocks: list[Block], + head: Mapping[str, str] | None = None) -> None: + if base_text is None or head_text is None: + issues.append(Issue("index_added_or_deleted", path, "index files are never created or deleted by a change set")) + return + directory = str(PurePosixPath(path).parent) + expected_new = sorted(PurePosixPath(p).name for p in new_pages + if str(PurePosixPath(p).parent) == directory + and p.endswith(".md") and PurePosixPath(p).name != INDEX_NAME) + # pages already in the directory but missing from the old index may be + # linked too (the release sweep's T1 repair); new pages MUST be + allowed = set(expected_new) | { + PurePosixPath(p).name for p in (head or {}) + if str(PurePosixPath(p).parent) == directory and p.endswith(".md") + and PurePosixPath(p).name != INDEX_NAME and f"]({PurePosixPath(p).name})" not in base_text + } + for name in expected_new: # required whatever else the index change does + if f"]({name})" not in head_text: + issues.append(Issue("index_missing", path, f"new page {name} is not linked")) + if not head_text.startswith(base_text.rstrip("\n") + "\n") and head_text != base_text: + blocks.append(Block("prose", path, "", "prose", _sha(head_text))) + return + added = head_text[len(base_text.rstrip("\n") + "\n"):].splitlines() + files = [] + for line in added: + match = _INDEX_LINE.fullmatch(line.strip()) + if match is None: + blocks.append(Block("prose", path, "", "prose", _sha(head_text))) + return + files.append(match.group("file")) + if not set(expected_new) <= set(files) or not set(files) <= allowed or len(files) != len(set(files)): + issues.append(Issue("index_mismatch", path, + f"index lines must cover new pages {expected_new} and only link unlisted pages; got {files}")) + + +def check_changeset( + base: Mapping[str, str], + head: Mapping[str, str], + changes: Iterable[Change], + *, + external_texts: Mapping[str, str] | None = None, + release: str = "", +) -> ChangesetResult: + """Gate one change set. ``base``/``head`` map knowledge-relative paths + (``repos/...``) to text; ``changes`` are repository-relative git changes. + ``external_texts`` (repo path -> text, outside ``knowledge/``) is scanned + for references to rules this change retires or purges. ``release`` is the + trusted current release (from the signed verdict); when given, purges must + carry it as ``purged_at``. A purge is always rejected in the release that + retired the rule.""" + changes = list(changes) + issues = _whitelist_issues(changes) + blocks: list[Block] = [] + retired: list[str] = [] + purged: list[tuple[str, str]] = [] # (rule id, page) + purge_retired_at: dict[str, str] = {} + knowledge_paths = [c.path[len(KNOWLEDGE_PREFIX):] for c in changes + if c.path.startswith(KNOWLEDGE_PREFIX)] + for path in knowledge_paths: + issues.extend(_text_issues(path, head.get(path))) + if path.endswith(".md") and head.get(path) is not None: + over = page_over_capacity(head[path]) + if over: + issues.append(Issue("page_over_capacity", path, over)) + new_pages = {p for p in knowledge_paths if p not in base and p in head} + for page_path in sorted(new_pages): + index = str(PurePosixPath(page_path).with_name(INDEX_NAME)) + if page_path.endswith(".md") and PurePosixPath(page_path).name != INDEX_NAME \ + and index not in knowledge_paths: + issues.append(Issue("index_missing", page_path, f"{index} must list the new page")) + tombstone_paths = [] + for path in sorted(knowledge_paths): + name = PurePosixPath(path).name + if name == INDEX_NAME: + _index_mechanical(path, base.get(path), head.get(path), new_pages, issues, blocks, head) + elif name == TOMBSTONES_NAME: + tombstone_paths.append(path) + elif name == ROUTES_NAME: + blocks.append(Block("prose", path, "", "prose", _sha(head.get(path) or ""))) + elif path.endswith(".md"): + try: + _classify_page(path, base.get(path), head.get(path), blocks, issues, + retired, purged, purge_retired_at) + except (LifecycleError, yaml.YAMLError) as exc: + issues.append(Issue("page_unparseable", path, str(exc))) + # tombstones: exactly the IDs purged in this change set, appended + for path in tombstone_paths: + try: + before = load_tombstones(base.get(path)) + after = load_tombstones(head.get(path)) + except (LifecycleError, yaml.YAMLError) as exc: + issues.append(Issue("tombstones_invalid", path, str(exc))) + continue + scope = str(PurePosixPath(path).parent) + purged_here = [rid for rid, page_path in purged if page_path.startswith(scope + "/")] + appended = after[len(before):] + if after[:len(before)] != before or sorted(i["id"] for i in appended) != sorted(purged_here): + issues.append(Issue("tombstones_mismatch", path, f"expected appended tombstones for {sorted(purged_here)}")) + for item in appended: + retired_at = purge_retired_at.get(str(item["id"])) + purged_at = str(item.get("purged_at") or "") + if retired_at is None: + continue # a nested ID; its parent carries the timing + if not purged_at or purged_at == retired_at: + issues.append(Issue("purge_too_early", path, + f"{item['id']}: purged in the release it was retired ({retired_at})")) + if release and purged_at != release: + issues.append(Issue("purge_release_mismatch", path, + f"{item['id']}: purged_at {purged_at} is not this release {release}")) + if head.get(path) != render_tombstones(after): + issues.append(Issue("tombstones_mismatch", path, "tombstones are not in canonical form")) + for rule_id, _page in purged: + if rule_id not in all_tombstoned_ids(head): + issues.append(Issue("purge_without_tombstone", "", rule_id)) + # no NEW duplicate IDs, no reuse of retired or tombstoned IDs + base_ids, head_ids = all_rule_ids(base), all_rule_ids(head) + for rule_id, count in head_ids.items(): + if count > base_ids.get(rule_id, 0) and count > 1: + issues.append(Issue("duplicate_rule_id", "", rule_id)) + base_tomb = all_tombstoned_ids(base) + for block in blocks: + if block.op == "add" and (block.rule_id in base_tomb or block.rule_id in base_ids): + issues.append(Issue("rule_id_reused", block.path, block.rule_id)) + # dangling references to what this change set retired or purged + gone = set(retired) | {rid for rid, _page in purged} + if gone: + issues.extend(_dangling_refs(head, gone)) + external = tuple( + (rule_id, path) + for path, text in sorted((external_texts or {}).items()) + for rule_id in sorted(gone) if re.search(rf"(? list[Issue]: + """References in ACTIVE text to rules that are now retired or purged. + Allowed: the rule's own section, lifecycle footers, tombstones.""" + issues = [] + for path, text in head.items(): + if not path.endswith(".md"): + continue + page = Page.parse(text) + chunks = [page.head] + [ + s.body_without_footer for s in page.sections + if s.rule_id not in gone and (not s.is_rule or _active(s)) + ] + body = "\n".join(chunks) + for rule_id in sorted(gone): + if re.search(rf"(? bool: + try: + return section.footer.status == "active" + except LifecycleError: + return True diff --git a/.github/kb-gate/lifecycle.py b/.github/kb-gate/lifecycle.py new file mode 100644 index 00000000..dafe6531 --- /dev/null +++ b/.github/kb-gate/lifecycle.py @@ -0,0 +1,424 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/lifecycle.py -- do not edit here. +"""Rule pages as data: sections, lifecycle footers, and byte-preserving render. + +A rule page is frontmatter, a head (title + preamble), then sections. A section +starts at every ``## `` heading, and also at a ``### — `` heading +whose enclosing ``## `` section is not itself a rule (owner pages group rules +under topic headings). A section whose heading is ``## <ID> — <title>`` or +``### <ID> — <title>`` is a rule; ``###`` rules INSIDE a ``##`` rule stay part +of it as nested rules. Each rule may end +with one machine-readable footer, an HTML comment that renders invisibly:: + + <!-- kb:rule status=retired since=v0.30.0 retired_at=v0.31.0 reason=upstream-removed evidence="PR #8201" superseded_by=BENCH-1c --> + +No footer means ``status=active``, which is every rule written before the +lifecycle existed. Parsing then rendering an untouched page returns its exact +bytes, so an operation changes only the sections it targets. + +This module needs only the standard library and PyYAML and imports nothing +else from the package: the ``kb-gate`` verifier bundle vendors it verbatim. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field, replace +from typing import Iterable + +import yaml + +RULE_HEADING = re.compile(r"^(?P<level>#{2,3})\s+(?P<rule>[A-Za-z0-9][A-Za-z0-9-]{1,40})\s+[—-]\s+\S") +_TOP_RULE = re.compile(r"^##\s+[A-Za-z0-9][A-Za-z0-9-]{1,40}\s+[—-]\s+\S") +ANY_RULE_HEADING = re.compile( + r"^#{2,6}\s+(?P<rule>[A-Za-z0-9][A-Za-z0-9-]{1,40})(?=\s|$)", re.MULTILINE) +FOOTER = re.compile(r"^<!--\s*kb:rule\s+(?P<attrs>.*?)\s*-->\s*$") +_ATTR = re.compile(r'(?P<key>[a-z_]+)=(?:"(?P<quoted>[^"]*)"|(?P<bare>[^\s"]+))') +FRONTMATTER = re.compile(r"\A---\r?\n(?P<body>.*?)\r?\n---(?:\r?\n|\Z)", re.DOTALL) +CITATION = re.compile(r"\^\[(?P<ref>[^\]\n]{1,60})\]") + +STATUSES = ("active", "retired") +FOOTER_KEYS = ( + "status", "since", "retired_at", "reason", "evidence", + "supersedes", "superseded_by", "protected", +) +RETIRE_REASONS = ("upstream-removed", "superseded", "incorrect", "duplicate") + + +class LifecycleError(ValueError): + """A page, footer, or operation violates the rule lifecycle contract.""" + + +@dataclass(frozen=True) +class Footer: + status: str = "active" + since: str = "" + retired_at: str = "" + reason: str = "" + evidence: str = "" + supersedes: str = "" + superseded_by: str = "" + protected: bool = False + + @classmethod + def parse(cls, attrs: str) -> "Footer": + values: dict[str, str] = {} + consumed = 0 + for match in _ATTR.finditer(attrs): + if attrs[consumed:match.start()].strip(): + raise LifecycleError(f"malformed kb:rule footer: {attrs!r}") + key = match.group("key") + if key not in FOOTER_KEYS: + raise LifecycleError(f"unknown kb:rule footer key: {key}") + if key in values: + raise LifecycleError(f"repeated kb:rule footer key: {key}") + values[key] = match.group("quoted") if match.group("quoted") is not None \ + else match.group("bare") + consumed = match.end() + if attrs[consumed:].strip(): + raise LifecycleError(f"malformed kb:rule footer: {attrs!r}") + status = values.get("status", "active") + if status not in STATUSES: + raise LifecycleError(f"kb:rule status must be one of {STATUSES}: {status}") + protected = values.get("protected", "false") + if protected not in {"true", "false"}: + raise LifecycleError("kb:rule protected must be true or false") + footer = cls( + status=status, + since=values.get("since", ""), + retired_at=values.get("retired_at", ""), + reason=values.get("reason", ""), + evidence=values.get("evidence", ""), + supersedes=values.get("supersedes", ""), + superseded_by=values.get("superseded_by", ""), + protected=protected == "true", + ) + footer.check() + return footer + + def check(self) -> None: + if self.status == "retired": + missing = [key for key in ("retired_at", "reason", "evidence") + if not getattr(self, key)] + if missing: + raise LifecycleError(f"retired rule footer lacks {', '.join(missing)}") + if self.reason not in RETIRE_REASONS: + raise LifecycleError( + f"retire reason must be one of {RETIRE_REASONS}: {self.reason}") + if self.reason == "superseded" and not self.superseded_by: + raise LifecycleError("a superseded rule must name superseded_by") + elif self.retired_at or self.superseded_by: + raise LifecycleError("an active rule cannot carry retired_at/superseded_by") + + def render(self) -> str: + parts = [] + for key in FOOTER_KEYS: + value = getattr(self, key) + if key == "protected": + if value: + parts.append("protected=true") + continue + if key == "status": + parts.append(f"status={value}") + continue + if not value: + continue + if re.search(r'[\s"]', value): + if '"' in value: + raise LifecycleError(f"footer value may not contain a quote: {key}") + parts.append(f'{key}="{value}"') + else: + parts.append(f"{key}={value}") + return f"<!-- kb:rule {' '.join(parts)} -->" + + +@dataclass(frozen=True) +class Section: + """One level-two section, kept as its exact source text.""" + + text: str # includes the heading line and its trailing newline(s) + + @property + def heading(self) -> str: + return self.text.split("\n", 1)[0] + + @property + def rule_id(self) -> str: + match = RULE_HEADING.match(self.heading) + return match.group("rule") if match else "" + + @property + def level(self) -> int: + match = RULE_HEADING.match(self.heading) + return len(match.group("level")) if match else 2 + + @property + def is_rule(self) -> bool: + return bool(self.rule_id) + + def _footer_line(self) -> tuple[int, str] | None: + lines = self.text.rstrip("\n").split("\n") + for index in range(len(lines) - 1, 0, -1): + line = lines[index].strip() + if not line: + continue + return (index, line) if FOOTER.match(line) else None + return None + + @property + def footer(self) -> Footer: + found = self._footer_line() + if found is None: + return Footer() + return Footer.parse(FOOTER.match(found[1]).group("attrs")) + + @property + def has_footer(self) -> bool: + return self._footer_line() is not None + + @property + def body_without_footer(self) -> str: + """The rule's words: the section minus its footer, trailing blanks trimmed.""" + found = self._footer_line() + lines = self.text.rstrip("\n").split("\n") + if found is not None: + lines = lines[:found[0]] + return "\n".join(lines).rstrip() + "\n" + + @property + def content_sha256(self) -> str: + return hashlib.sha256(self.body_without_footer.encode("utf-8")).hexdigest() + + @property + def nested_rule_ids(self) -> tuple[str, ...]: + return tuple(m.group("rule") for m in ANY_RULE_HEADING.finditer(self.text)) + + @property + def citations(self) -> tuple[str, ...]: + return tuple(dict.fromkeys(m.group("ref").strip() for m in CITATION.finditer(self.text))) + + def with_footer(self, footer: Footer) -> "Section": + footer.check() + trailing = self.text[len(self.text.rstrip("\n")):] or "\n" + return Section(self.body_without_footer + "\n" + footer.render() + trailing) + + +@dataclass(frozen=True) +class Page: + frontmatter: str # the exact "---...---\n" block, or "" when absent + head: str # text between frontmatter and the first "## " section + sections: tuple[Section, ...] = field(default_factory=tuple) + + @classmethod + def parse(cls, text: str) -> "Page": + match = FRONTMATTER.match(text) + frontmatter = match.group(0) if match else "" + rest = text[len(frontmatter):] + chunks = _split_sections(rest) + head = chunks[0] + sections = tuple(Section(chunk) for chunk in chunks[1:]) + return cls(frontmatter, head, sections) + + def render(self) -> str: + return self.frontmatter + self.head + "".join(s.text for s in self.sections) + + # -- rules --------------------------------------------------------------- + + def rules(self) -> tuple[Section, ...]: + return tuple(s for s in self.sections if s.is_rule) + + def rule(self, rule_id: str) -> Section: + for section in self.sections: + if section.rule_id == rule_id: + return section + raise LifecycleError(f"rule {rule_id} is not on this page") + + def has_rule(self, rule_id: str) -> bool: + return any(section.rule_id == rule_id for section in self.sections) + + def replace_section(self, rule_id: str, new: Section | None) -> "Page": + out = [] + found = False + for section in self.sections: + if section.rule_id == rule_id: + found = True + if new is not None: + out.append(new) + else: + out.append(section) + if not found: + raise LifecycleError(f"rule {rule_id} is not on this page") + return replace(self, sections=tuple(out)) + + def append_section(self, section: Section) -> "Page": + sections = list(self.sections) + if sections and not sections[-1].text.endswith("\n\n"): + last = sections[-1].text + sections[-1] = Section(last.rstrip("\n") + "\n\n") + elif not sections and self.head and not self.head.endswith("\n\n"): + return replace(self, head=self.head.rstrip("\n") + "\n\n", sections=(section,)) + sections.append(section) + return replace(self, sections=tuple(sections)) + + # -- frontmatter --------------------------------------------------------- + + def frontmatter_field(self, key: str) -> str | None: + match = re.search(rf"(?m)^{re.escape(key)}:[ \t]*(?P<value>.*)$", self.frontmatter) + return match.group("value").strip() if match else None + + def with_frontmatter_field(self, key: str, value: str) -> "Page": + if not self.frontmatter: + raise LifecycleError("page has no frontmatter") + pattern = re.compile(rf"(?m)^{re.escape(key)}:[^\r\n]*$") + if not pattern.search(self.frontmatter): + raise LifecycleError(f"frontmatter has no {key} field") + return replace(self, frontmatter=pattern.sub(f"{key}: {value}", self.frontmatter, count=1)) + + def frontmatter_data(self) -> dict: + match = FRONTMATTER.match(self.frontmatter) + if not match: + return {} + try: + data = yaml.safe_load(match.group("body")) or {} + except yaml.YAMLError as exc: + raise LifecycleError(f"frontmatter is not valid YAML: {exc}") from exc + if not isinstance(data, dict): + raise LifecycleError("frontmatter must be a mapping") + return data + + def sources(self) -> list[str]: + value = self.frontmatter_data().get("sources") + if value is None: + return [] + if not isinstance(value, list): + raise LifecycleError("frontmatter sources must be a list") + return [str(item) for item in value] + + def with_sources(self, items: list[str]) -> "Page": + """Rewrite ``sources:`` in the page's own style (flow or block list).""" + match = re.search( + r"(?m)^sources:[ \t]*(?P<flow>\[.*\])?[ \t]*\r?\n(?P<block>(?:[ \t]+-[^\r\n]*\r?\n)*)", + self.frontmatter, + ) + if match is None: + raise LifecycleError("frontmatter has no sources field") + if match.group("flow") is not None or not match.group("block"): + rendered = f"sources: {render_flow_list(items)}\n" + else: + indent = re.match(r"[ \t]+", match.group("block")).group(0) + rendered = "sources:\n" + "".join( + f'{indent}- "{_quote_free(item)}"\n' for item in items) + return replace( + self, + frontmatter=self.frontmatter[:match.start()] + rendered + + self.frontmatter[match.end():], + ) + + +_FENCE = re.compile(r"^\s{0,3}(`{3,}|~{3,})") + + +def _split_sections(text: str) -> list[str]: + """Split at ``## `` headings, and at every ``### `` heading whose enclosing + ``## `` section is not a rule (so a level-three rule ends at its next + sibling); ``###`` inside a ``##`` rule stays nested. Fences never split.""" + chunks: list[str] = [] + current: list[str] = [] + fence: str | None = None + top_is_rule = False + for line in text.splitlines(keepends=True): + match = _FENCE.match(line) + if match: + marker = match.group(1) + if fence is None: + fence = marker[0] * len(marker) + elif marker[0] == fence[0] and len(marker) >= len(fence): + fence = None + split = False + if fence is None and not match: + if line.startswith("## "): + split = True + top_is_rule = bool(_TOP_RULE.match(line)) + elif line.startswith("### ") and not top_is_rule: + # every ### under a topic heading is its own section, so a + # level-three rule ends at its next sibling heading + split = True + if split: + chunks.append("".join(current)) + current = [line] + else: + current.append(line) + chunks.append("".join(current)) + return chunks + + +def parse_flow_list(raw: str) -> list[str]: + """Parse the YAML flow sequence the tree uses for ``sources:``/``tags:``.""" + raw = raw.strip() + if not (raw.startswith("[") and raw.endswith("]")): + raise LifecycleError(f"expected a [flow, list]: {raw!r}") + inner = raw[1:-1].strip() + if not inner: + return [] + items: list[str] = [] + for match in re.finditer(r'\s*(?:"(?P<q>[^"]*)"|(?P<b>[^,]+?))\s*(?:,|$)', inner): + value = match.group("q") if match.group("q") is not None else match.group("b") + if value is not None and value.strip(): + items.append(value.strip()) + return items + + +def _quote_free(item: str) -> str: + if '"' in item: + raise LifecycleError(f"list item may not contain a quote: {item!r}") + return item + + +def render_flow_list(items: Iterable[str]) -> str: + rendered = [] + for item in items: + if re.fullmatch(r"[A-Za-z0-9_./:@+-]+", item): + rendered.append(item) + else: + if '"' in item: + raise LifecycleError(f"list item may not contain a quote: {item!r}") + rendered.append(f'"{item}"') + return "[" + ", ".join(rendered) + "]" + + +def expected_sources(before: list[str], page: Page) -> list[str]: + """Deterministic ``sources:`` after a change: the old list, in order, plus + every citation an ACTIVE rule on the page makes that is not yet listed, in + order of first appearance. Nothing is ever removed (history stays cited).""" + out = list(before) + for section in page.rules(): + if section.footer.status != "active": + continue + for ref in section.citations: + if ref not in out: + out.append(ref) + return out + + +def visible_text(text: str) -> str: + """The page as reviewers and agents should see it: retired rules removed. + + A retired rule keeps its text on disk for one release cycle so the + retirement can be reviewed and reverted, but it must stop being SERVED at + once. Pages without retired rules are returned unchanged, byte for byte. + """ + if "kb:rule" not in text: + return text + page = Page.parse(text) + kept = [] + for section in page.sections: + try: + retired = section.is_rule and section.footer.status == "retired" + except LifecycleError: + retired = False # L1 rejects malformed footers; never hide on doubt + if not retired: + kept.append(section) + if len(kept) == len(page.sections): + return text + return replace(page, sections=tuple(kept)).render() + diff --git a/.github/kb-gate/ops.py b/.github/kb-gate/ops.py new file mode 100644 index 00000000..7e18421d --- /dev/null +++ b/.github/kb-gate/ops.py @@ -0,0 +1,376 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/ops.py -- do not edit here. +"""Knowledge Ops API 2.0: typed changes to rule pages, with the lifecycle. + +The v1 curation path (``KnowledgeCurator``) can only append a section. The +knowledge service needs every change a maintainer makes by hand, expressed as a +typed operation the quality gate can judge one by one: + +==================== ========================================================= +``add`` append a new rule (optionally on a new ``rules-*.md`` page) +``edit_same_meaning`` reword an active rule under the same ID (gate must confirm + the meaning is unchanged) +``replace`` retire an active rule as ``superseded`` and add its + successor under a NEW ID, atomically +``retire`` retire an active rule (upstream removed it, it was wrong, + or it duplicates another); its text stays for one cycle +``purge`` delete a retired rule's text and tombstone its ID forever +==================== ========================================================= + +``apply_operations`` is a pure function over a mapping of knowledge-relative +paths to text, so the service, the gate and the verifier all compute exactly +the same result. After the operations it writes the MECHANICAL consequences the +verifier later recomputes: ``updated:``, ``sources:`` (append-only), an index +line for every new page, and tombstones for purged IDs. + +Standard library + PyYAML + ``lifecycle`` only: vendored by the verifier bundle. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import PurePosixPath +from typing import Iterable, Mapping + +import yaml + +try: # package import; the verifier bundle vendors both modules side by side + from .lifecycle import ( + ANY_RULE_HEADING, RETIRE_REASONS, RULE_HEADING, Footer, LifecycleError, + Page, Section, expected_sources, + ) +except ImportError: # pragma: no cover - exercised by the vendored bundle + from lifecycle import ( # type: ignore[no-redef] + ANY_RULE_HEADING, RETIRE_REASONS, RULE_HEADING, Footer, LifecycleError, + Page, Section, expected_sources, + ) + +KNOWLEDGE_OPS_API_VERSION = "2.0.0" +OP_KINDS = ("add", "edit_same_meaning", "replace", "retire", "purge") +TOMBSTONES_NAME = "_tombstones.yaml" +INDEX_NAME = "_index.md" +_RULE_PAGE = re.compile(r"rules(?:-[a-z0-9][a-z0-9-]*)?\.md") +_DATE = re.compile(r"\d{4}-\d{2}-\d{2}") +_MAX_SECTION_CHARS = 16 * 1024 +# Mirrors knowledge/tools/check_knowledge_tree.py's split gate (>= 32 KiB or +# >= 500 non-empty lines fails the whole tree). +PAGE_MAX_BYTES = 32 * 1024 +PAGE_MAX_LINES = 500 + + +def page_over_capacity(text: str) -> str: + size = len(text.encode("utf-8")) + lines = sum(1 for line in text.splitlines() if line.strip()) + if size >= PAGE_MAX_BYTES or lines >= PAGE_MAX_LINES: + return f"{size} bytes / {lines} non-empty lines (limit < {PAGE_MAX_BYTES} / < {PAGE_MAX_LINES})" + return "" + + +@dataclass(frozen=True) +class KnowledgeOperation: + kind: str + page: str + rule_id: str + section_markdown: str = "" + new_rule_id: str = "" + new_page: str = "" + reason: str = "" + evidence: str = "" + page_title: str = "" + allow_protected: bool = False + + def to_dict(self) -> dict: + return {key: value for key, value in self.__dict__.items() if value not in ("", False)} + + @classmethod + def from_dict(cls, data: Mapping) -> "KnowledgeOperation": + unknown = set(data) - set(cls.__dataclass_fields__) + if unknown: + raise LifecycleError(f"unknown operation fields: {sorted(unknown)}") + return cls(**{key: data[key] for key in data}) + + +@dataclass(frozen=True) +class OperationsResult: + files: dict[str, str] # every created or changed path -> new text + touched_rules: tuple[str, ...] # rule IDs whose sections changed + created_pages: tuple[str, ...] = field(default_factory=tuple) + + +def repo_scope(page: str) -> str: + """``repos/<repo>`` or ``general`` for a knowledge-relative page path.""" + parts = PurePosixPath(page).parts + if len(parts) >= 2 and parts[0] == "repos": + return f"repos/{parts[1]}" + if parts and parts[0] == "general": + return "general" + raise LifecycleError(f"page is outside repos/<repo>/ and general/: {page}") + + +def tombstones_path(page: str) -> str: + return f"{repo_scope(page)}/{TOMBSTONES_NAME}" + + +def load_tombstones(text: str | None) -> list[dict]: + if not text: + return [] + data = yaml.safe_load(text) or {} + if not isinstance(data, dict) or data.get("schema_version") != 1: + raise LifecycleError("tombstones must be schema_version 1") + items = data.get("ids") or [] + if not isinstance(items, list) or any( + not isinstance(item, dict) or not item.get("id") for item in items + ): + raise LifecycleError("tombstones ids must be a list of {id, ...}") + return items + + +def render_tombstones(items: list[dict]) -> str: + lines = [ + "# IDs of purged knowledge rules. An ID listed here is never reused.", + "# Written by the knowledge service; recomputed by the kb-gate verifier.", + "schema_version: 1", + "ids:", + ] + for item in items: + lines.append( + f" - {{id: {item['id']}, purged_at: {item.get('purged_at', '')}, " + f"page: {item.get('page', '')}}}" + ) + return "\n".join(lines) + "\n" + + +def all_rule_ids(files: Mapping[str, str]) -> dict[str, int]: + """Heading IDs at any level across every rule page, with their counts.""" + counts: dict[str, int] = {} + for path, text in files.items(): + if not path.endswith(".md"): + continue + for match in ANY_RULE_HEADING.finditer(_without_fences(text)): + counts[match.group("rule")] = counts.get(match.group("rule"), 0) + 1 + return counts + + +def all_tombstoned_ids(files: Mapping[str, str]) -> set[str]: + out: set[str] = set() + for path, text in files.items(): + if PurePosixPath(path).name == TOMBSTONES_NAME: + out.update(str(item["id"]) for item in load_tombstones(text)) + return out + + +def _without_fences(text: str) -> str: + out, fence = [], None + for line in text.splitlines(keepends=True): + stripped = line.lstrip() + if stripped.startswith(("```", "~~~")): + marker = stripped[:3] + fence = None if fence == marker else (fence or marker) + out.append("\n") + continue + out.append("\n" if fence else line) + return "".join(out) + + +def _check_section(section_markdown: str, rule_id: str, level: int = 2) -> Section: + text = section_markdown.strip("\n") + "\n" + section = Section(text) + marks = "#" * level + if section.rule_id != rule_id or section.level != level: + raise LifecycleError(f"section heading must be '{marks} {rule_id} — <title>'") + if level == 2 and len(re.findall(r"(?m)^## ", text)) != 1: + raise LifecycleError("a rule section has exactly one level-two heading") + if level == 3 and (re.search(r"(?m)^## ", text) or len( + [m for m in re.finditer(r"(?m)^### ", text)]) != 1): + raise LifecycleError("a level-three rule section has exactly one heading and no ## heading") + if section.has_footer: + raise LifecycleError("the service writes lifecycle footers; do not include one") + if not 80 <= len(text) <= _MAX_SECTION_CHARS: + raise LifecycleError("section length is outside 80..16384 characters") + return section + + +def _new_page_text(page: str, title: str, today: str, template: Page | None) -> str: + tags = "[]" + if template is not None and template.frontmatter_field("tags") is not None: + tags = template.frontmatter_field("tags") + if not title or '"' in title: + raise LifecycleError("a new page needs a title without quotes") + return ( + "---\n" + f'title: "{title}"\n' + f"created: {today}\n" + f"updated: {today}\n" + "type: rule\n" + f"tags: {tags}\n" + "sources: []\n" + "---\n\n" + f"# {title}\n\n" + ) + + +def index_line(page: str, title: str) -> str: + return f"- [{title}]({PurePosixPath(page).name})\n" + + +def apply_operations( + files: Mapping[str, str], + operations: Iterable[KnowledgeOperation], + *, + release: str, + today: str, +) -> OperationsResult: + """Apply typed operations to ``files`` (knowledge-relative path -> text). + + Raises ``LifecycleError`` on the first invalid operation; nothing is + partially applied because the input mapping is never mutated. + """ + if not _DATE.fullmatch(today): + raise LifecycleError("today must be YYYY-MM-DD") + if not release or re.search(r'[\s"]', release): + raise LifecycleError("release must be a non-empty token") + work: dict[str, str] = dict(files) + original_ids = all_rule_ids(files) + tombstoned = all_tombstoned_ids(files) + added_ids: set[str] = set() + touched_pages: dict[str, list[str]] = {} # page -> sources before any change + created: list[str] = [] + touched_rules: list[str] = [] + purged: dict[str, list[dict]] = {} + + def page_of(path: str, *, create_title: str = "") -> Page: + if not _RULE_PAGE.fullmatch(PurePosixPath(path).name): + raise LifecycleError(f"not a rule page (rules.md / rules-<topic>.md): {path}") + repo_scope(path) + text = work.get(path) + if text is None: + if not create_title: + raise LifecycleError(f"page does not exist: {path}") + sibling = str(PurePosixPath(path).with_name("rules.md")) + template = Page.parse(work[sibling]) if sibling in work else None + index = str(PurePosixPath(path).with_name(INDEX_NAME)) + if index not in work: + raise LifecycleError(f"a new page's directory needs an {INDEX_NAME}: {index}") + text = _new_page_text(path, create_title, today, template) + work[path] = text + created.append(path) + work[index] = work[index].rstrip("\n") + "\n" + index_line(path, create_title) + page = Page.parse(text) + if page.frontmatter_field("type") != "rule": + raise LifecycleError(f"page is not type: rule: {path}") + touched_pages.setdefault(path, page.sources() if path not in created else []) + return page + + def fresh_id(rule_id: str) -> None: + if rule_id in original_ids or rule_id in added_ids: + raise LifecycleError(f"rule ID already exists in the tree: {rule_id}") + if rule_id in tombstoned: + raise LifecycleError(f"rule ID was purged and is reserved forever: {rule_id}") + added_ids.add(rule_id) + + def active(page: Page, rule_id: str, op: KnowledgeOperation) -> Section: + section = page.rule(rule_id) + footer = section.footer + if footer.status != "active": + raise LifecycleError(f"rule {rule_id} is not active") + if footer.protected and not op.allow_protected: + raise LifecycleError(f"rule {rule_id} is protected; only the human path may change it") + return section + + for op in operations: + if op.kind not in OP_KINDS: + raise LifecycleError(f"unknown operation kind: {op.kind}") + if op.kind == "add": + section = _check_section(op.section_markdown, op.rule_id) + fresh_id(op.rule_id) + for nested in section.nested_rule_ids: + if nested != op.rule_id: + fresh_id(nested) + page = page_of(op.page, create_title=op.page_title) + page = page.append_section(section.with_footer(Footer(status="active", since=release))) + work[op.page] = page.render() + touched_rules.append(op.rule_id) + elif op.kind == "edit_same_meaning": + page = page_of(op.page) + old = active(page, op.rule_id, op) + section = _check_section(op.section_markdown, op.rule_id, old.level) + if set(section.nested_rule_ids) != set(old.nested_rule_ids): + raise LifecycleError("an edit may not add or drop nested rule IDs") + trailing = old.text[len(old.text.rstrip("\n")):] + new = Section(section.text.rstrip("\n") + (trailing or "\n")) + if old.has_footer: + new = new.with_footer(old.footer) + work[op.page] = page.replace_section(op.rule_id, new).render() + touched_rules.append(op.rule_id) + elif op.kind == "replace": + if not op.new_rule_id or op.new_rule_id == op.rule_id: + raise LifecycleError("replace needs a different new_rule_id") + if not op.evidence: + raise LifecycleError("replace needs evidence") + page = page_of(op.page) + old = active(page, op.rule_id, op) + section = _check_section(op.section_markdown, op.new_rule_id) + fresh_id(op.new_rule_id) + for nested in section.nested_rule_ids: + if nested != op.new_rule_id: + fresh_id(nested) + retired = old.with_footer(Footer( + status="retired", since=old.footer.since, retired_at=release, + reason="superseded", evidence=op.evidence, + supersedes=old.footer.supersedes, # keep the chain A -> B -> C + superseded_by=op.new_rule_id, protected=old.footer.protected, + )) + work[op.page] = page.replace_section(op.rule_id, retired).render() + target = op.new_page or op.page + target_page = page_of(target, create_title=op.page_title) + successor = section.with_footer(Footer( + status="active", since=release, supersedes=op.rule_id)) + work[target] = target_page.append_section(successor).render() + touched_rules += [op.rule_id, op.new_rule_id] + elif op.kind == "retire": + if op.reason not in RETIRE_REASONS or op.reason == "superseded": + raise LifecycleError("retire reason must be upstream-removed, incorrect or duplicate; use replace for superseded") + if not op.evidence: + raise LifecycleError("retire needs evidence") + page = page_of(op.page) + old = active(page, op.rule_id, op) + retired = old.with_footer(Footer( + status="retired", since=old.footer.since, retired_at=release, + reason=op.reason, evidence=op.evidence, + supersedes=old.footer.supersedes, protected=old.footer.protected, + )) + work[op.page] = page.replace_section(op.rule_id, retired).render() + touched_rules.append(op.rule_id) + elif op.kind == "purge": + page = page_of(op.page) + old = page.rule(op.rule_id) + if old.footer.status != "retired": + raise LifecycleError(f"only a retired rule can be purged: {op.rule_id}") + if old.footer.protected and not op.allow_protected: + raise LifecycleError(f"rule {op.rule_id} is protected; only the human path may purge it") + if old.footer.retired_at == release: + raise LifecycleError("a rule is purged one full release after it was retired") + work[op.page] = page.replace_section(op.rule_id, None).render() + for rid in dict.fromkeys((op.rule_id, *old.nested_rule_ids)): + purged.setdefault(tombstones_path(op.page), []).append( + {"id": rid, "purged_at": release, "page": op.page}) + touched_rules.append(op.rule_id) + + for path, before_sources in touched_pages.items(): + page = Page.parse(work[path]) + page = page.with_frontmatter_field("updated", today) + page = page.with_sources(expected_sources(before_sources, page)) + work[path] = page.render() + for path, entries in purged.items(): + work[path] = render_tombstones(load_tombstones(work.get(path)) + entries) + for path in touched_pages: + over = page_over_capacity(work[path]) + if over: + raise LifecycleError(f"page full, add to a new rules-<topic>.md page instead: {path}: {over}") + + changed = {path: text for path, text in work.items() if files.get(path) != text} + return OperationsResult( + files=changed, + touched_rules=tuple(dict.fromkeys(touched_rules)), + created_pages=tuple(created), + ) diff --git a/.github/kb-gate/requirements.lock b/.github/kb-gate/requirements.lock new file mode 100644 index 00000000..895725e5 --- /dev/null +++ b/.github/kb-gate/requirements.lock @@ -0,0 +1,227 @@ +# GENERATED by tools/build_kb_gate_bundle.py --lock. Install with: +# pip install --require-hashes --no-deps -r .github/kb-gate/requirements.lock +cryptography==50.0.1 \ + --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ + --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ + --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ + --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ + --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ + --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ + --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ + --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ + --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ + --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ + --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ + --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ + --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ + --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ + --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ + --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ + --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ + --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ + --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ + --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ + --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ + --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ + --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ + --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ + --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ + --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ + --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ + --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ + --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ + --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ + --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ + --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ + --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ + --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ + --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ + --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ + --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ + --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ + --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ + --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ + --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ + --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ + --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ + --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ + --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ + --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 +PyYAML==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 diff --git a/.github/kb-gate/signing.py b/.github/kb-gate/signing.py new file mode 100644 index 00000000..9c1934a6 --- /dev/null +++ b/.github/kb-gate/signing.py @@ -0,0 +1,118 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/signing.py -- do not edit here. +"""Ed25519 envelopes for knowledge-gate verdicts, outbox items and control records. + +An envelope is ``{"payload": <object>, "key_id": <str>, "signature": <b64>}``. +The signature covers the canonical JSON of the payload (sorted keys, compact +separators, UTF-8) prefixed with a purpose tag, so a verdict signature can never +be replayed as an outbox item or a control record and vice versa. + +Only ``cryptography`` beyond the standard library: the kb-gate verifier bundle +vendors this module and pins that dependency by hash. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +from pathlib import Path +from typing import Any + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, Ed25519PublicKey, +) + +PURPOSES = ("kb-gate-verdict", "kb-outbox-item", "kb-control", "kb-holds", "kb-ack") + + +class SignatureError(ValueError): + """An envelope is malformed, signed for another purpose, or forged.""" + + +def canonical_json(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _message(purpose: str, payload: Any) -> bytes: + if purpose not in PURPOSES: + raise SignatureError(f"unknown signature purpose: {purpose}") + return purpose.encode("ascii") + b"\0" + canonical_json(payload) + + +def public_key_id(public: Ed25519PublicKey) -> str: + raw = public.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return "ed25519:" + hashlib.sha256(raw).hexdigest()[:16] + + +def public_key_text(public: Ed25519PublicKey) -> str: + """The one-line form committed as ``.github/kb-gate.pub``.""" + raw = public.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + return "ed25519 " + base64.b64encode(raw).decode("ascii") + + +def load_public_key(text: str) -> Ed25519PublicKey: + parts = text.strip().split() + if len(parts) != 2 or parts[0] != "ed25519": + raise SignatureError("public key must be 'ed25519 <base64>'") + try: + return Ed25519PublicKey.from_public_bytes(base64.b64decode(parts[1], validate=True)) + except (ValueError, TypeError) as exc: + raise SignatureError("public key is not a valid Ed25519 key") from exc + + +def generate_private_key(path: str | Path) -> Ed25519PrivateKey: + """Create a new private key file readable only by its owner.""" + path = Path(path) + if path.exists(): + raise SignatureError(f"refusing to overwrite an existing key: {path}") + key = Ed25519PrivateKey.generate() + data = key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(data) + return key + + +def load_private_key(path: str | Path) -> Ed25519PrivateKey: + path = Path(path) + mode = path.stat().st_mode & 0o077 + if mode: + raise SignatureError(f"private key must not be readable by group/others: {path}") + key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(key, Ed25519PrivateKey): + raise SignatureError("private key is not Ed25519") + return key + + +def sign(purpose: str, payload: Any, key: Ed25519PrivateKey) -> dict: + signature = key.sign(_message(purpose, payload)) + return { + "purpose": purpose, + "payload": payload, + "key_id": public_key_id(key.public_key()), + "signature": base64.b64encode(signature).decode("ascii"), + } + + +def verify(purpose: str, envelope: Any, public: Ed25519PublicKey) -> Any: + """Return the payload of a valid envelope for ``purpose``; raise otherwise.""" + if not isinstance(envelope, dict) or set(envelope) != {"purpose", "payload", "key_id", "signature"}: + raise SignatureError("envelope must have exactly purpose, payload, key_id and signature") + if envelope["purpose"] != purpose: + raise SignatureError(f"envelope is signed for {envelope['purpose']!r}, not {purpose!r}") + if envelope["key_id"] != public_key_id(public): + raise SignatureError("envelope was signed by a different key") + try: + signature = base64.b64decode(envelope["signature"], validate=True) + public.verify(signature, _message(purpose, envelope["payload"])) + except (InvalidSignature, ValueError, TypeError) as exc: + raise SignatureError("signature does not verify") from exc + return envelope["payload"] diff --git a/.github/kb-gate/verdict.py b/.github/kb-gate/verdict.py new file mode 100644 index 00000000..59f44c4c --- /dev/null +++ b/.github/kb-gate/verdict.py @@ -0,0 +1,149 @@ +# GENERATED by tools/build_kb_gate_bundle.py from src/infermatrix_copilot/knowledge_service/verdict.py -- do not edit here. +"""The kb-gate verdict: what the knowledge service signs and the verifier checks. + +A verdict binds a decision to ONE pull request state: + +* identity: repository, PR number, exact head SHA, nonce, issue time and a + 72-hour issue window (it must first verify within it); +* the complete patch manifest: every changed path with change type, pre-image + and post-image git blob IDs and file modes, compared independently of any + base SHA (so a queued PR is checked against what the queue actually applies); +* the per-block judgements for every rule and prose block L1 produced, the + change-set consistency judgement with the page hashes it saw, and the + context base those judgements were made against; +* upstream fact attestations (the verifier re-checks public upstreams); +* who decided: ``auto`` (L1 + L2 + consistency) or ``human-approved`` (with the + approving review IDs and reviewers). + +Standard library only (plus ``signing`` for the envelope): vendored by kb-gate. +""" + +from __future__ import annotations + +import hashlib +import re +import secrets +from dataclasses import dataclass +from typing import Any, Iterable, Mapping + +SCHEMA = "kb-gate-verdict/1" +ISSUE_WINDOW_SECONDS = 72 * 3600 +SOURCES = ("auto", "human-approved") +_SHA40 = re.compile(r"[0-9a-f]{40}") + + +class VerdictError(ValueError): + """A verdict is malformed or does not match what it claims to cover.""" + + +def git_blob_id(data: bytes | str | None) -> str: + """The git object ID of a blob with this content ("" for an absent file).""" + if data is None: + return "" + if isinstance(data, str): + data = data.encode("utf-8") + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + + +@dataclass(frozen=True) +class ManifestEntry: + path: str + status: str # A | M | D + old_blob: str + new_blob: str + old_mode: str + new_mode: str + + def to_dict(self) -> dict: + return {"path": self.path, "status": self.status, "old_blob": self.old_blob, + "new_blob": self.new_blob, "old_mode": self.old_mode, "new_mode": self.new_mode} + + +def manifest_from_files(base: Mapping[str, str], head: Mapping[str, str], *, + prefix: str = "knowledge/") -> list[dict]: + """Patch manifest for a change expressed as knowledge-relative text maps.""" + entries = [] + for rel in sorted(set(base) | set(head)): + before, after = base.get(rel), head.get(rel) + if before == after: + continue + status = "A" if before is None else "D" if after is None else "M" + entries.append(ManifestEntry( + path=prefix + rel, status=status, + old_blob=git_blob_id(before), new_blob=git_blob_id(after), + old_mode="" if before is None else "100644", + new_mode="" if after is None else "100644", + ).to_dict()) + return entries + + +def canonical_manifest(entries: Iterable[Mapping[str, Any]]) -> list[dict]: + """Sorted, field-complete manifest; the comparison form (no base SHA).""" + out = [] + for entry in entries: + missing = {"path", "status", "old_blob", "new_blob", "old_mode", "new_mode"} - set(entry) + if missing: + raise VerdictError(f"manifest entry lacks {sorted(missing)}") + out.append({key: str(entry[key]) for key in + ("path", "status", "old_blob", "new_blob", "old_mode", "new_mode")}) + return sorted(out, key=lambda item: item["path"]) + + +def build_verdict(*, repository: str, pr: int, head_sha: str, context_base_sha: str, + manifest: list[dict], blocks: list[dict], consistency: list[dict], + release: str, upstream: dict, facts: list[dict], models: dict, + source: str = "auto", review_ids: tuple[int, ...] = (), + reviewers: tuple[str, ...] = (), issued_at: float) -> dict: + if source not in SOURCES: + raise VerdictError(f"source must be one of {SOURCES}") + if not _SHA40.fullmatch(head_sha) or not _SHA40.fullmatch(context_base_sha): + raise VerdictError("head_sha and context_base_sha must be 40-character hex SHAs") + if source == "human-approved" and not (review_ids and reviewers): + raise VerdictError("a human-approved verdict names its approving reviews") + for block in blocks: + if block.get("verdict") != "pass": + raise VerdictError(f"only passing blocks are signed: {block.get('block_id')}") + for item in consistency: + if item.get("verdict") != "consistent": + raise VerdictError("only a consistent change set is signed") + return { + "schema": SCHEMA, + "repository": repository, + "pr": int(pr), + "head_sha": head_sha, + "context_base_sha": context_base_sha, + "issued_at": float(issued_at), + "valid_until": float(issued_at) + ISSUE_WINDOW_SECONDS, + "nonce": secrets.token_hex(16), + "source": source, + "review_ids": [int(value) for value in review_ids], + "reviewers": list(reviewers), + "release": release, + "upstream": upstream, + "manifest": canonical_manifest(manifest), + "blocks": sorted(blocks, key=lambda b: b["block_id"]), + "consistency": consistency, + "facts": facts, + "models": models, + } + + +def check_binding(verdict: Mapping[str, Any], *, repository: str, pr: int, head_sha: str, + now: float) -> None: + """Identity half of verification: the verdict names THIS PR state and is + within its issue window. Replaying a verdict on another PR, another head, + or after the window fails here.""" + if verdict.get("schema") != SCHEMA: + raise VerdictError("unknown verdict schema") + if verdict.get("repository") != repository or int(verdict.get("pr", -1)) != int(pr): + raise VerdictError("verdict is for another repository or pull request") + if verdict.get("head_sha") != head_sha: + raise VerdictError("verdict is for another head commit") + if not float(verdict.get("issued_at", 0)) <= now <= float(verdict.get("valid_until", 0)): + raise VerdictError("verdict is outside its issue window") + if verdict.get("source") not in SOURCES: + raise VerdictError("verdict source is invalid") + + +def manifests_equal(signed: Iterable[Mapping[str, Any]], actual: Iterable[Mapping[str, Any]]) -> bool: + return canonical_manifest(signed) == canonical_manifest(actual) diff --git a/.github/workflows/kb-gate.yml b/.github/workflows/kb-gate.yml new file mode 100644 index 00000000..85dcfb15 --- /dev/null +++ b/.github/workflows/kb-gate.yml @@ -0,0 +1,99 @@ +name: kb-gate + +# The required check for knowledge pull requests (doc/features/knowledge-service.md). +# +# Two stages, one check name: +# * precheck (pull_request_target / issue_comment / workflow_dispatch): verifies +# the PR against current main and posts the `kb-gate` commit STATUS on the PR +# head. It only decides whether the PR may enter the merge queue. +# * merge group (merge_group): the job named `kb-gate` verifies every segment of +# the group on the exact commit that would land. This decides what lands. +# +# Security: the precheck runs with the BASE branch's workflow and bundle +# (pull_request_target), reads the PR only as git objects, never checks it out, +# never installs or executes anything from it, and holds no secret (the repository +# is public, so PR refs are fetched anonymously; the token only reads the API and +# posts the status). Both jobs +# run the verifier ONLY from .github/kb-gate/ (pinned, hash-locked deps). + +on: + pull_request_target: + types: [opened, synchronize, reopened, labeled, unlabeled, edited, ready_for_review, converted_to_draft] + issue_comment: + types: [created, edited] + workflow_dispatch: + inputs: + pr: + description: PR number to (re)check + required: true + merge_group: + types: [checks_requested] + +permissions: + contents: read + +jobs: + precheck: + if: >- + github.event_name == 'pull_request_target' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + contains(github.event.comment.body, '<!-- kb-gate:verdict:v1 -->')) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + statuses: write + concurrency: + group: kb-gate-${{ github.event.pull_request.number || github.event.issue.number || inputs.pr }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned verifier + run: | + cd .github/kb-gate && sha256sum --check --strict MANIFEST.sha256 + python -m venv "$RUNNER_TEMP/kb-gate" + "$RUNNER_TEMP/kb-gate/bin/pip" install --quiet --require-hashes --no-deps -r requirements.lock + - name: Verify and post the kb-gate status + env: + GITHUB_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number || github.event.issue.number || inputs.pr }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + "$RUNNER_TEMP/kb-gate/bin/python" -E -s .github/kb-gate/kb_gate.py pr \ + --repository "$GITHUB_REPOSITORY" --pr "$PR" --post-status --target-url "$RUN_URL" + + kb-gate: + if: github.event_name == 'merge_group' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned verifier + run: | + cd .github/kb-gate && sha256sum --check --strict MANIFEST.sha256 + python -m venv "$RUNNER_TEMP/kb-gate" + "$RUNNER_TEMP/kb-gate/bin/pip" install --quiet --require-hashes --no-deps -r requirements.lock + - name: Verify every segment of the merge group + env: + GITHUB_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.merge_group.head_sha }} + BASE_REF: ${{ github.event.merge_group.base_ref }} + run: | + "$RUNNER_TEMP/kb-gate/bin/python" -E -s .github/kb-gate/kb_gate.py merge-group \ + --repository "$GITHUB_REPOSITORY" --head-sha "$HEAD_SHA" --base-ref "${BASE_REF#refs/heads/}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c65dc5c..e22e731f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,7 @@ name: tests on: push: pull_request: + merge_group: workflow_dispatch: permissions: @@ -183,6 +184,12 @@ jobs: - name: SPEC coverage and freshness run: python tools/check_spec_freshness.py --strict + # A reminder, not a gate: a stale bundle keeps kb-gate at its previous + # strictness. Regenerating it is a .github/** change (CODEOWNERS review). + - name: kb-gate bundle matches its source + continue-on-error: true + run: python tools/build_kb_gate_bundle.py --check + # Written, tested, and until now invoked only from prose (AGENTS.md, DEVELOPMENT.md). - name: Knowledge linters run: | diff --git a/doc/architecture/SPEC/knowledge_service.md b/doc/architecture/SPEC/knowledge_service.md index 9917de17..eec2f95c 100644 --- a/doc/architecture/SPEC/knowledge_service.md +++ b/doc/architecture/SPEC/knowledge_service.md @@ -47,3 +47,26 @@ PR, head or window. Standard library only; vendored by kb-gate. ## 2026-09-28 T1 索引修复 L1 的索引检查允许为目录中已存在但未列入索引的页面补链接（巡检 T1 修复），新页面仍必须被链接。 + +## 2026-09-28 kb-gate 验证器 +`gate_verifier` 是仓库端必需检查 `kb-gate`。`tools/build_kb_gate_bundle.py` 把它和 +`l1`、`lifecycle`、`ops`、`signing`、`verdict` 原样复制到 `.github/kb-gate/`（附 +`requirements.lock` 按哈希固定依赖，`MANIFEST.sha256` 为包内文件哈希），工作流只从该目录运行。 +- PR 阶段（`pull_request_target`/带判定标记的 `issue_comment`/手动）：用 git 对象读取 PR（从不检出 + 或执行），在 `merge-tree` 结果上跑 L1，把 `kb-gate` 状态发布到**已验证的**那个 head。 +- merge group 阶段：`main..head` 的 first-parent 链每段必须是双亲合并提交；第二父按 open PR 的 + head 唯一反查；每个触碰 `knowledge/` 的段以其第一父为有效 base 独立验证，最后在整棵落地树上跑 L1。 +- 知识变更通过条件：签名有效且绑定本仓库/PR/head、在签发有效期内；清单与实际变更（与 base 无关） + 相等；`auto` 要求块表与 L1 分块完全一致且全部 pass、一致性判定覆盖相同 owner 目录且页面哈希与落地树 + 相同、`context_base_sha..有效 base` 未改动上下文集合；`human-approved` 要求绑定的审阅仍为 + APPROVED、针对当前 head、由 `.github/kb-gate/` 的 CODEOWNERS（知识维护者）之一给出，且标签仍在。 + 另需无 `kb:hold`、签名暂停清单新鲜（≤10 分钟）且未命中；清单缺失/伪造/过期一律失败。 + `human-approved` 豁免自动合并白名单（`knowledge/tools`、`skills`、脚本、混合 PR），页面生命周期 L1 只作用于 + 受治理页面，其余 L1 不豁免； + 本变更退役/删除的规则 ID 在落地树中不得被引用（排在后面的 PR 或 main 可能新增了引用）。 + 暂停清单按范围匹配：`repos/<r>/` 对应仓库名 `<r>`，`general/` 对应 `general`。 +- 不变量：失败即关闭——缺公钥、缺 `holds_url`、无法归属的 merge group、带上游事实证明的判定都失败； + 未触碰 `knowledge/` 的变更不读取任何判定或暂停清单，总是通过。git < 2.38 时 PR 阶段用临时索引做 + 只接受单侧改动的三方合并（更严格，不更宽松）。 +- 测试：`test_kb_gate_verifier.py`（重放/过期/伪造、清单漏项、块表不全、混合路径与可执行位、暂停清单、 + 上下文失效、状态发布、人工审批撤销、merge group 逐段与前序同目录 PR、验证包新鲜与自包含、独立运行）。 diff --git a/doc/features/knowledge-service.md b/doc/features/knowledge-service.md index 128faac9..5bf87c69 100644 --- a/doc/features/knowledge-service.md +++ b/doc/features/knowledge-service.md @@ -64,3 +64,28 @@ T1/T2/T3 与 purge，每个规则页一个变更集，全部经过质量门。 评审服务读取知识：将 `KNOWLEDGE_ROOT` 指向 `$KB_STATE_DIR/active`。 + +## 仓库端门禁 `kb-gate` + +必需检查 `kb-gate`（`.github/workflows/kb-gate.yml`）只从受 CODEOWNERS 保护的验证包 +`.github/kb-gate/` 运行，从不导入 `src/`、`tools/`、`knowledge/tools/`。 + +- **PR 预检**：`pull_request_target`、带判定标记的评论或手动触发；在当前 `main` 上验证后， + 把 `kb-gate` 状态发布到 PR head。只决定能否入队。 +- **merge group**：名为 `kb-gate` 的作业逐段验证将要落地的提交；失败则 PR 被移出队列。 +- 未触碰 `knowledge/` 的 PR 总是通过；知识 PR 需要有效签名判定、清单一致、L1 通过、 + 暂停清单新鲜且未命中（见 SPEC `knowledge_service.md` 的 kb-gate 验证器一节）。 + +修改验证逻辑后重新生成验证包（属于 `.github/**` 改动，需要知识维护者审批）： + +```bash +python tools/build_kb_gate_bundle.py # 重新生成 +python tools/build_kb_gate_bundle.py --check # CI 提醒（非阻塞） +python tools/build_kb_gate_bundle.py --lock # 从 PyPI 重新按哈希固定依赖 +``` + +切换前的负责人步骤：提交 `.github/kb-gate.pub`（`kb keygen` 的公钥）；在 +`.github/kb-gate/config.json` 填入暂停清单的公开 HTTPS 地址（`$KB_STATE_DIR/public/holds.json`）； +在 `.github/CODEOWNERS` 中再加入至少一名知识维护者；仓库 admin 开启合并队列（merge commit） +与 ruleset（必需检查 `kb-gate` 限定 GitHub Actions 来源、Code Owner 审阅、新推送需重新批准）。 +在此之前，所有触碰 `knowledge/` 的 PR 都会被 `kb-gate` 拒绝（失败即关闭），代码 PR 不受影响。 diff --git a/src/infermatrix_copilot/knowledge_service/gate_verifier.py b/src/infermatrix_copilot/knowledge_service/gate_verifier.py new file mode 100644 index 00000000..d90224a5 --- /dev/null +++ b/src/infermatrix_copilot/knowledge_service/gate_verifier.py @@ -0,0 +1,691 @@ +"""kb-gate: the repository-side verifier for knowledge pull requests. + +It is the required check ``kb-gate`` and runs in two places under one name: + +* PR stage (``pull_request_target`` / ``issue_comment`` / dispatch): verifies + the PR against the current base and posts a ``kb-gate`` commit status on the + PR head. It only decides whether a PR may ENTER the merge queue. +* merge group stage (``merge_group``): verifies every segment of the group on + the exact commit that would land and exits non-zero on any failure. This is + what decides whether anything lands. + +A change that does not touch ``knowledge/`` passes. A knowledge change passes +only when all of these hold for it: + +1. L1: whitelisted paths only, regular files only, no new tree issues, on the + PR's own change AND on the tree that would land; +2. a valid Ed25519-signed verdict comment for this repository, PR and exact + head SHA, inside its issue window (the comment's author is irrelevant); +3. the signed patch manifest equals the change as applied (base independent); +4. ``auto``: the signed block table equals the blocks L1 derives, every block + passed, the consistency judgements cover the same owner directories and + their page hashes match the tree that would land; no page in the context + set changed between the judged base and the effective base; + ``human-approved``: every bound review is still APPROVED on this head, one + is by a knowledge maintainer (a CODEOWNERS owner of ``.github/kb-gate/``), + and ``kb:human-approved`` is set; +5. no ``kb:hold`` label and not held by the service's signed hold list (a + missing, forged or stale hold list fails closed). + +PR content is only ever READ through git objects; nothing from the PR is +executed or installed. This module is vendored into ``.github/kb-gate/`` by +``tools/build_kb_gate_bundle.py`` and runs from there, so it imports nothing +outside the standard library and its vendored siblings. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Iterable, Mapping + +try: + from .l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree + from .lifecycle import LifecycleError, Page + from .ops import all_rule_ids, repo_scope + from .signing import SignatureError, load_public_key, verify + from .verdict import VerdictError, check_binding, manifests_equal +except ImportError: # pragma: no cover - vendored verifier bundle + from l1 import Change, _dangling_refs as dangling_refs, check_changeset, check_tree # type: ignore[no-redef] + from lifecycle import LifecycleError, Page # type: ignore[no-redef] + from ops import all_rule_ids, repo_scope # type: ignore[no-redef] + from signing import SignatureError, load_public_key, verify # type: ignore[no-redef] + from verdict import VerdictError, check_binding, manifests_equal # type: ignore[no-redef] + +STATUS_CONTEXT = "kb-gate" +VERDICT_MARKER = "<!-- kb-gate:verdict:v1 -->" +HOLD_LABEL = "kb:hold" +HUMAN_LABEL = "kb:human-approved" +HOLDS_MAX_AGE = 10 * 60 +MAINTAINER_PATH = ".github/kb-gate/kb_gate.py" +WHITELIST_CODES = ("path_not_whitelisted", "mode_not_regular") +KNOWLEDGE_PREFIX = "knowledge/" +GOVERNED_PREFIXES = ("knowledge/repos/", "knowledge/general/") +GOVERNED_SUFFIXES = (".md", ".yaml") +EXTERNAL_REF_DIRS = ("skills/", "plugins/", "adapters/", "doc/", "playbooks/") +EXTERNAL_SUFFIXES = (".md", ".yaml", ".yml", ".json", ".txt") +_ZERO = "0" * 40 + + +class GateError(RuntimeError): + """The verifier could not establish a fact it needs (fails closed).""" + + +# -- git (objects only; nothing from the PR is ever checked out or executed) --- + +class Git: + def __init__(self, cwd: str | Path): + self.cwd = str(cwd) + + def run(self, *args: str, input: bytes | None = None, ok: tuple[int, ...] = (0,)) -> bytes: + proc = subprocess.run(["git", *args], cwd=self.cwd, input=input, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode not in ok: + raise GateError(f"git {' '.join(args)} failed: {proc.stderr.decode(errors='replace')[:400]}") + return proc.stdout + + def rev(self, name: str) -> str: + return self.run("rev-parse", "--verify", "--quiet", f"{name}^{{commit}}").decode().strip() + + def tree_of(self, name: str) -> str: + return self.run("rev-parse", "--verify", f"{name}^{{tree}}").decode().strip() + + def parents(self, sha: str) -> list[str]: + return self.run("rev-list", "--parents", "-n", "1", sha).decode().split()[1:] + + def is_ancestor(self, older: str, newer: str) -> bool: + proc = subprocess.run(["git", "merge-base", "--is-ancestor", older, newer], cwd=self.cwd, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, check=False) + if proc.returncode not in (0, 1): + raise GateError(f"cannot relate {older[:12]} and {newer[:12]}: " + f"{proc.stderr.decode(errors='replace')[:200]}") + return proc.returncode == 0 + + def merge_base(self, a: str, b: str) -> str: + return self.run("merge-base", a, b).decode().strip() + + def first_parent_chain(self, base: str, head: str) -> list[str]: + return self.run("rev-list", "--first-parent", "--reverse", f"{base}..{head}").decode().split() + + def merge_tree(self, base: str, head: str) -> str: + """The tree a merge of ``head`` into ``base`` would produce (PR stage).""" + proc = subprocess.run(["git", "merge-tree", "--write-tree", base, head], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + if proc.returncode != 0 and b"--write-tree" in proc.stderr: # git < 2.38: no --write-tree + return self._index_merge(base, head) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().split()[0] + + def _index_merge(self, base: str, head: str) -> str: + """Only merges where each path changed on at most one side; anything + else is reported as not merging cleanly (stricter, never looser).""" + import tempfile + + with tempfile.TemporaryDirectory() as scratch: + env = {**os.environ, "GIT_INDEX_FILE": str(Path(scratch) / "index")} + for args in (["read-tree", "-i", "-m", self.merge_base(base, head), base, head], ["write-tree"]): + proc = subprocess.run(["git", *args], cwd=self.cwd, env=env, check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if proc.returncode != 0: + raise GateError("the PR does not merge cleanly into the current base") + return proc.stdout.decode().strip() + + def raw_diff(self, old: str, new: str) -> list[dict]: + """Every changed path as a manifest entry (renames are D + A).""" + out = self.run("diff", "--raw", "-z", "--no-renames", "--no-abbrev", old, new).split(b"\0") + entries: list[dict] = [] + index = 0 + while index < len(out) - 1: + meta = out[index].decode() + path = out[index + 1].decode("utf-8", "replace") + index += 2 + if not meta.startswith(":"): + raise GateError(f"unexpected git diff output: {meta!r}") + old_mode, new_mode, old_blob, new_blob, status = meta[1:].split() + entries.append({ + "path": path, "status": status[0], + "old_blob": "" if old_blob == _ZERO else old_blob, + "new_blob": "" if new_blob == _ZERO else new_blob, + "old_mode": "" if old_mode == "000000" else old_mode, + "new_mode": "" if new_mode == "000000" else new_mode, + }) + return entries + + def changed_names(self, old: str, new: str) -> list[str]: + out = self.run("diff", "--name-only", "-z", "--no-renames", old, new, "--", "knowledge/") + return [name.decode("utf-8", "replace") for name in out.split(b"\0") if name] + + def texts(self, rev: str, prefixes: tuple[str, ...], suffixes: tuple[str, ...]) -> dict[str, str]: + listing = self.run("ls-tree", "-r", "-z", "--full-tree", rev).split(b"\0") + wanted: list[tuple[str, str]] = [] + for entry in listing: + if not entry: + continue + meta, _, raw_path = entry.partition(b"\t") + mode, kind, oid = meta.decode().split() + name = raw_path.decode("utf-8", "replace") + if kind == "blob" and mode == "100644" and name.startswith(prefixes) and name.endswith(suffixes): + wanted.append((name, oid)) + if not wanted: + return {} + out = self.run("cat-file", "--batch", input="".join(f"{oid}\n" for _, oid in wanted).encode()) + texts: dict[str, str] = {} + cursor = 0 + for name, _oid in wanted: + header_end = out.index(b"\n", cursor) + size = int(out[cursor:header_end].split()[2]) + body = out[header_end + 1: header_end + 1 + size] + cursor = header_end + 1 + size + 1 + try: + texts[name] = body.decode("utf-8") + except UnicodeDecodeError: + continue + return texts + + def knowledge_files(self, rev: str) -> dict[str, str]: + files = self.texts(rev, GOVERNED_PREFIXES, GOVERNED_SUFFIXES) + return {name[len(KNOWLEDGE_PREFIX):]: text for name, text in files.items()} + + def external_texts(self, rev: str) -> dict[str, str]: + return self.texts(rev, EXTERNAL_REF_DIRS, EXTERNAL_SUFFIXES) + + def show(self, rev: str, path: str) -> str | None: + proc = subprocess.run(["git", "show", f"{rev}:{path}"], cwd=self.cwd, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + return proc.stdout.decode("utf-8", "replace") if proc.returncode == 0 else None + + +# -- GitHub REST (read, plus the PR-stage status) -------------------------------- + +class GitHub: + def __init__(self, repository: str, token: str, api: str = "https://api.github.com"): + self.repository, self._token, self._api = repository, token, api.rstrip("/") + + def _request(self, method: str, path: str, body: dict | None = None) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(self._api + path, data=data, method=method, headers={ + "Accept": "application/vnd.github+json", "Authorization": f"Bearer {self._token}", + "X-GitHub-Api-Version": "2022-11-28", "User-Agent": "kb-gate"}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read() or b"null") + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"GitHub {method} {path} failed: {exc}") from exc + + def get(self, path: str) -> Any: + return self._request("GET", path) + + def get_all(self, path: str, *, pages: int = 10) -> list: + out: list = [] + sep = "&" if "?" in path else "?" + for page in range(1, pages + 1): + batch = self.get(f"{path}{sep}per_page=100&page={page}") + out.extend(batch) + if len(batch) < 100: + break + return out + + def post(self, path: str, body: dict) -> Any: + return self._request("POST", path, body) + + +# -- inputs from the protected checkout ------------------------------------------- + +def fetch_holds(url: str) -> Any: + if not url: + raise GateError("no hold-list URL is configured (.github/kb-gate/config.json)") + try: + with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "kb-gate"}), + timeout=20) as response: + return json.loads(response.read()) + except (urllib.error.URLError, TimeoutError, ValueError) as exc: + raise GateError(f"hold list unreachable: {exc}") from exc + + +def verify_holds(envelope: Any, public_key, *, now: float) -> dict: + holds = verify("kb-holds", envelope, public_key) + if not 0 <= now - float(holds["issued_at"]) <= HOLDS_MAX_AGE: + raise GateError("hold list is stale (the knowledge service may be down)") + return holds + + +def parse_codeowners(text: str | None) -> list[tuple[str, tuple[str, ...]]]: + rules = [] + for line in (text or "").splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + pattern, *owners = line.split() + rules.append((pattern, tuple(owner.lstrip("@").lower() for owner in owners))) + return rules + + +def _codeowner_match(pattern: str, path: str) -> bool: + anchored = pattern.startswith("/") + pattern = pattern.lstrip("/") + if pattern.endswith("/"): + pattern += "**" + candidates = [path] if anchored or "/" in pattern.rstrip("*") else \ + [path] + [path.split("/", i)[-1] for i in range(1, path.count("/") + 1)] + for candidate in candidates: + if fnmatch.fnmatchcase(candidate, pattern) or fnmatch.fnmatchcase(candidate, pattern + "/**") \ + or (pattern.endswith("/**") and candidate.startswith(pattern[:-2])): + return True + return False + + +def code_owners(rules: list[tuple[str, tuple[str, ...]]], path: str) -> tuple[str, ...]: + """Owners of ``path``: the LAST matching line wins. Team entries never + match a person here (their membership is not readable with the job token), + so a team-only path cannot be human-approved through this verifier.""" + owners: tuple[str, ...] = () + for pattern, names in rules: + if _codeowner_match(pattern, path): + owners = names + return tuple(name for name in owners if "/" not in name) + + +# -- the verification of one change ------------------------------------------------ + +@dataclass +class Context: + git: Git + github: Any + repository: str + public_key: Any # None when .github/kb-gate.pub is missing + holds_loader: Callable[[], Any] + codeowners: list[tuple[str, tuple[str, ...]]] + now: float + _holds: dict | None = None + + def holds(self) -> dict: + if self._holds is None: + self._holds = verify_holds(self.holds_loader(), self.public_key, now=self.now) + return self._holds + + +def governed(path: str) -> bool: + """A page the knowledge lifecycle governs (what ``knowledge_files`` reads).""" + return path.startswith(GOVERNED_PREFIXES) and path.endswith(GOVERNED_SUFFIXES) + + +def touches_knowledge(entries: Iterable[Mapping[str, str]]) -> bool: + return any(entry["path"].startswith(KNOWLEDGE_PREFIX) for entry in entries) + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def verdict_envelopes(comments: Iterable[Mapping[str, Any]]) -> list[Any]: + out = [] + for comment in comments: + body = str(comment.get("body") or "") + if VERDICT_MARKER not in body: + continue + rest = body.split(VERDICT_MARKER, 1)[1] + start = rest.find("```json") + if start < 0: + continue + rest = rest[start + len("```json"):] + end = rest.find("```") + try: + out.append(json.loads(rest[:end] if end >= 0 else rest)) + except ValueError: + continue + return out + + +def find_verdict(ctx: Context, pr: int, head_sha: str) -> tuple[dict | None, list[str]]: + """The newest valid verdict for exactly this PR head, and why others failed.""" + comments = ctx.github.get_all(f"/repos/{ctx.repository}/issues/{pr}/comments") + valid, rejected = [], [] + for envelope in verdict_envelopes(comments): + try: + verdict = verify("kb-gate-verdict", envelope, ctx.public_key) + check_binding(verdict, repository=ctx.repository, pr=pr, head_sha=head_sha, now=ctx.now) + valid.append(verdict) + except (SignatureError, VerdictError, KeyError, TypeError, ValueError) as exc: + rejected.append(str(exc)) + if not valid: + return None, rejected + return max(valid, key=lambda v: float(v["issued_at"])), rejected + + +def _owner_dir(path: str) -> str: + return str(PurePosixPath(path).parent) + + +def _scope_root(path: str) -> str: + parts = PurePosixPath(path).parts + return "/".join(parts[:2]) if parts[:1] == ("repos",) else parts[0] + + +def context_paths(touched: set[str], owner_dirs: set[str], post: Mapping[str, str], + base: Mapping[str, str]) -> Callable[[str], bool]: + """The pages whose change since the judged base invalidates L2: the touched + pages, every page in their owner directories, the scope's _routes.yaml and + _index.md files (scope root and owner directories), and pages holding rules + the change supersedes.""" + roots = {_scope_root(path) for path in touched} + dirs = owner_dirs | {_owner_dir(path) for path in touched} + superseded: set[str] = set() + for path in touched: + text = post.get(path) + if not text or not path.endswith(".md"): + continue + try: + sections = Page.parse(text).rules() + except (LifecycleError, ValueError): + continue + for section in sections: + try: + target = section.footer.supersedes + except LifecycleError: + continue + for rule_id in [t.strip() for t in target.split(",") if t.strip()]: + superseded.update(p for p, t in base.items() + if p.endswith(".md") and rule_id in all_rule_ids({p: t})) + + def relevant(path: str) -> bool: + name = PurePosixPath(path).name + parent = _owner_dir(path) + return (path in touched or path in superseded or parent in dirs + or (name in ("_routes.yaml", "_index.md") and parent in roots | dirs)) + return relevant + + +def verify_change(ctx: Context, *, pr: dict, head_sha: str, pre: str, post: str, final: str, + effective_base: str, stage: str) -> list[str]: + """Problems with one PR's knowledge change (empty when it passes). + + ``pre``/``post`` are the commits (or trees) around this PR's change as it + is applied; ``final`` is the tree that would land; ``effective_base`` is + the state the change is applied on (current main, or the first parent of + the merge-group segment).""" + number = int(pr["number"]) + entries = ctx.git.raw_diff(pre, post) + if not touches_knowledge(entries): + return [] + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + problems: list[str] = [] + labels = {str(label.get("name")) for label in pr.get("labels") or []} + if pr.get("state") != "open": + problems.append(f"PR #{number} is not open") + if str((pr.get("head") or {}).get("sha")) != head_sha: + problems.append(f"PR #{number} head moved from {head_sha[:12]}") + if stage == "merge_group" and pr.get("draft"): + problems.append(f"PR #{number} is a draft (paused)") + if HOLD_LABEL in labels: + problems.append(f"PR #{number} carries {HOLD_LABEL}") + + pre_files, post_files = ctx.git.knowledge_files(pre), ctx.git.knowledge_files(post) + final_files = ctx.git.knowledge_files(final) + changes = [Change(e["path"], e["status"] if e["status"] in "AMD" else "M", e["old_mode"], e["new_mode"]) + for e in entries] + for e in entries: + if e["status"] not in ("A", "M", "D"): + problems.append(f"{e['path']}: change type {e['status']} is not allowed") + touched = {e["path"][len(KNOWLEDGE_PREFIX):] for e in entries if e["path"].startswith(KNOWLEDGE_PREFIX)} + + try: + holds = ctx.holds() + if holds.get("global"): + problems.append("the knowledge service holds every repository (global pause)") + if number in {int(p) for p in holds.get("prs") or []}: + problems.append(f"PR #{number} is on the service's hold list") + held = set(holds.get("repos") or []) + # a scope is its repository name, or "general" for knowledge/general/ + scopes = {repo_scope(p).split("/")[-1] for p in touched if p.startswith(("repos/", "general/"))} + for scope in sorted(scopes & held): + problems.append(f"repository {scope} is paused by the knowledge service") + except (GateError, SignatureError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems.append(f"hold list: {exc}") + + verdict, rejected = find_verdict(ctx, number, head_sha) + if verdict is None: + detail = f" (rejected: {'; '.join(rejected[:3])})" if rejected else "" + problems.append(f"no valid signed verdict for PR #{number} at {head_sha[:12]}{detail}") + # still report L1, so the author sees every structural problem at once + result = check_changeset(pre_files, post_files, changes, external_texts={}) + return problems + [f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues] + + if not manifests_equal(verdict.get("manifest") or [], entries): + problems.append("the signed patch manifest does not match the change as applied") + if verdict.get("facts"): + problems.append("the verdict carries upstream fact attestations this verifier cannot re-check") + + human = verdict["source"] == "human-approved" + # a maintainer's approval replaces the auto-merge whitelist (knowledge/tools, + # skills, scripts, mixed PRs) and L2, never the rest of L1: non-governed + # paths are not pages, so the page lifecycle checks run on governed ones only + l1_changes = [c for c in changes if governed(c.path)] if human else changes + result = check_changeset(pre_files, post_files, l1_changes, + external_texts=ctx.git.external_texts(final), + release=str(verdict.get("release") or "")) + waived = WHITELIST_CODES if human else () + problems.extend(f"L1 {i.code} {i.path} {i.detail}".rstrip() for i in result.issues if i.code not in waived) + # what this change retires or purges must not be cited anywhere in the tree + # that would land (a PR queued behind it, or main, may have added a citation) + gone = set(result.retired) | set(result.purged) + if gone: + try: + problems.extend(f"L1 {i.code} {i.path} {i.detail} (in the tree that would land)" + for i in dangling_refs(final_files, gone)) + except (LifecycleError, ValueError) as exc: + problems.append(f"L1 the tree that would land does not parse: {exc}") + + if verdict["source"] == "auto": + if result.external_refs: + problems.append("retired rules are still cited outside knowledge/ (needs a companion PR)") + signed_blocks = verdict.get("blocks") or [] + if {b.block_id for b in result.blocks} != {b.get("block_id") for b in signed_blocks}: + problems.append("the signed block table does not cover exactly the changed blocks") + if any(b.get("verdict") != "pass" for b in signed_blocks): + problems.append("a signed block did not pass") + owner_dirs = {_owner_dir(b.path) for b in result.blocks} + consistency = verdict.get("consistency") or [] + if {c.get("owner_dir") for c in consistency} != owner_dirs: + problems.append("the consistency judgement does not cover exactly the changed owner directories") + for item in consistency: + directory = item.get("owner_dir") + actual = {p: _sha256(t) for p, t in final_files.items() + if _owner_dir(p) == directory and p.endswith(".md")} + if item.get("verdict") != "consistent": + problems.append(f"{directory}: consistency was not judged consistent") + if item.get("pages") != actual: + problems.append(f"{directory}: pages changed since the consistency judgement") + base_sha = str(verdict["context_base_sha"]) + try: + if not ctx.git.is_ancestor(base_sha, effective_base): + problems.append("the judged context base is not an ancestor of the effective base") + else: + relevant = context_paths(touched, owner_dirs, post_files, pre_files) + moved = [n[len(KNOWLEDGE_PREFIX):] for n in ctx.git.changed_names(base_sha, effective_base)] + stale = sorted(p for p in moved if relevant(p)) + if stale: + problems.append("context changed since the verdict was judged: " + ", ".join(stale[:5])) + except GateError as exc: + problems.append(f"context base: {exc}") + else: + problems.extend(_human_approval_problems(ctx, pr, verdict, head_sha, labels)) + return problems + + +def _human_approval_problems(ctx: Context, pr: dict, verdict: dict, head_sha: str, + labels: set[str]) -> list[str]: + problems = [] + if HUMAN_LABEL not in labels: + problems.append(f"{HUMAN_LABEL} was removed") + number = int(pr["number"]) + reviewers = {str(r).lower() for r in verdict.get("reviewers") or []} + approvers = set() + for review_id in verdict.get("review_ids") or []: + review = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}/reviews/{int(review_id)}") + login = str((review.get("user") or {}).get("login") or "").lower() + if review.get("state") != "APPROVED": + problems.append(f"review {review_id} is no longer APPROVED") + elif review.get("commit_id") != head_sha: + problems.append(f"review {review_id} approved another head") + elif login not in reviewers: + problems.append(f"review {review_id} is by someone the verdict does not name") + else: + approvers.add(login) + # knowledge maintainers = the CODEOWNERS of the gate itself. Governed pages + # deliberately have no code owners (that would force a review on every auto + # PR); paths outside the whitelist get GitHub's native code-owner review. + maintainers = set(code_owners(ctx.codeowners, MAINTAINER_PATH)) + if not approvers & maintainers: + problems.append("no valid approving review by a knowledge maintainer (CODEOWNERS of .github/kb-gate/)") + return problems + + +def tree_problems(git: Git, base: str, final: str) -> list[str]: + """L1 on the tree that would land: no tree-level issue that base lacks.""" + before = {(i.code, i.detail) for i in check_tree(git.knowledge_files(base))} + return [f"L1 {i.code} {i.path} {i.detail}".rstrip() + for i in check_tree(git.knowledge_files(final)) if (i.code, i.detail) not in before] + + +# -- the two stages ----------------------------------------------------------------- + +def verify_pr(ctx: Context, number: int, *, base_ref: str = "main", + remote: str = "origin") -> tuple[str, list[str]]: + """(the head SHA verified, its problems). The status goes on THAT head, so + a push during verification can only leave the new head without a status.""" + pr = ctx.github.get(f"/repos/{ctx.repository}/pulls/{number}") + head = str(pr["head"]["sha"]) + return head, _verify_pr_head(ctx, pr, head, base_ref=base_ref, remote=remote) + + +def _verify_pr_head(ctx: Context, pr: dict, head: str, *, base_ref: str, remote: str) -> list[str]: + number = int(pr["number"]) + ctx.git.run("fetch", "--quiet", "--no-tags", remote, + f"+refs/pull/{number}/head:refs/kb-gate/pr-{number}", f"+refs/heads/{base_ref}:refs/kb-gate/base") + if ctx.git.rev(f"refs/kb-gate/pr-{number}") != head: + return [f"fetched head does not match PR #{number}'s head {head[:12]} (retry)"] + base_tip = ctx.git.rev("refs/kb-gate/base") + merge_base = ctx.git.merge_base(base_tip, head) + if not touches_knowledge(ctx.git.raw_diff(merge_base, head)): + return [] + try: + final = ctx.git.merge_tree(base_tip, head) + except GateError as exc: + return [str(exc)] + problems = verify_change(ctx, pr=pr, head_sha=head, pre=merge_base, post=head, final=final, + effective_base=base_tip, stage="pull_request") + return problems + tree_problems(ctx.git, base_tip, final) + + +def _pr_for_head(ctx: Context, sha: str) -> dict: + matches = [pr for pr in ctx.github.get_all(f"/repos/{ctx.repository}/pulls?state=open") + if str((pr.get("head") or {}).get("sha")) == sha] + if len(matches) != 1: + raise GateError(f"{len(matches)} open PRs have head {sha[:12]}; the group cannot be attributed") + return ctx.github.get(f"/repos/{ctx.repository}/pulls/{int(matches[0]['number'])}") + + +def verify_merge_group(ctx: Context, head_sha: str, *, base_ref: str = "main", + remote: str = "origin") -> list[str]: + """Every segment of the group, each against its own first parent.""" + ctx.git.run("fetch", "--quiet", "--no-tags", remote, f"+refs/heads/{base_ref}:refs/kb-gate/base") + main_tip = ctx.git.rev("refs/kb-gate/base") + if not ctx.git.is_ancestor(main_tip, head_sha): + return [f"the merge group {head_sha[:12]} is not built on the current {base_ref}"] + chain = ctx.git.first_parent_chain(main_tip, head_sha) + if not chain or chain[-1] != head_sha: + return ["the merge group has no segments"] + problems: list[str] = [] + any_knowledge = False + for commit in chain: + parents = ctx.git.parents(commit) + if len(parents) != 2: + return [f"segment {commit[:12]} is not a two-parent merge commit; the group cannot be attributed"] + first, second = parents + if not touches_knowledge(ctx.git.raw_diff(first, commit)): + continue + any_knowledge = True + if ctx.public_key is None: + return [".github/kb-gate.pub is not committed: knowledge changes cannot be verified"] + try: + pr = _pr_for_head(ctx, second) + except GateError as exc: + return [str(exc)] + found = verify_change(ctx, pr=pr, head_sha=second, pre=first, post=commit, final=head_sha, + effective_base=first, stage="merge_group") + problems.extend(f"PR #{pr['number']}: {p}" for p in found) + if any_knowledge: + problems.extend(tree_problems(ctx.git, main_tip, head_sha)) + return problems + + +# -- entry point ---------------------------------------------------------------------- + +def _load_context(args, github) -> Context: + root = Path(args.root) + config_path = Path(__file__).resolve().parent / "config.json" + config = json.loads(config_path.read_text()) if config_path.is_file() else {} + pub = root / ".github" / "kb-gate.pub" + public_key = load_public_key(pub.read_text()) if pub.is_file() else None + codeowners_path = root / ".github" / "CODEOWNERS" + holds_url = str(config.get("holds_url") or "") + return Context( + git=Git(root), github=github, repository=args.repository, public_key=public_key, + holds_loader=lambda: fetch_holds(holds_url), + codeowners=parse_codeowners(codeowners_path.read_text() if codeowners_path.is_file() else ""), + now=time.time(), + ) + + +def main(argv: list[str] | None = None, *, github=None) -> int: + parser = argparse.ArgumentParser(prog="kb-gate") + sub = parser.add_subparsers(dest="stage", required=True) + pr_stage = sub.add_parser("pr") + pr_stage.add_argument("--pr", type=int, required=True) + pr_stage.add_argument("--post-status", action="store_true") + pr_stage.add_argument("--target-url", default="") + group = sub.add_parser("merge-group") + group.add_argument("--head-sha", required=True) + for stage in (pr_stage, group): + stage.add_argument("--repository", required=True) + stage.add_argument("--root", default=".") + stage.add_argument("--base-ref", default="main") + args = parser.parse_args(argv) + github = github or GitHub(args.repository, os.environ.get("GITHUB_TOKEN", ""), + os.environ.get("GITHUB_API_URL", "https://api.github.com")) + ctx = _load_context(args, github) + head = "" + try: + if args.stage == "pr": + head = str(github.get(f"/repos/{args.repository}/pulls/{args.pr}")["head"]["sha"]) + head, problems = verify_pr(ctx, args.pr, base_ref=args.base_ref) + else: + problems = verify_merge_group(ctx, args.head_sha, base_ref=args.base_ref) + except (GateError, SignatureError, VerdictError, LifecycleError, KeyError, TypeError, ValueError) as exc: + problems = [f"verifier error: {exc}"] + for problem in problems: + print(f"kb-gate: {problem}") + print("kb-gate: " + ("FAIL" if problems else "PASS")) + if args.stage == "pr" and args.post_status and head: + description = (problems[0] if problems else "knowledge verdict verified")[:139] + github.post(f"/repos/{args.repository}/statuses/{head}", { + "state": "failure" if problems else "success", "context": STATUS_CONTEXT, + "description": description, **({"target_url": args.target_url} if args.target_url else {})}) + return 0 + return 1 if problems else 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/test/test_kb_gate_verifier.py b/test/test_kb_gate_verifier.py new file mode 100644 index 00000000..cc6b9343 --- /dev/null +++ b/test/test_kb_gate_verifier.py @@ -0,0 +1,484 @@ +"""The kb-gate verifier (PR precheck and merge group) and its pinned bundle, offline.""" + +from __future__ import annotations + +import ast +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from infermatrix_copilot.kb_service.gate import changes_between, run_gate, signable_blocks +from infermatrix_copilot.kb_service.models import ModelReply, ModelRole +from infermatrix_copilot.knowledge_service import gate_verifier as gv +from infermatrix_copilot.knowledge_service.ops import KnowledgeOperation as Op, apply_operations +from infermatrix_copilot.knowledge_service.signing import public_key_text, sign +from infermatrix_copilot.knowledge_service.verdict import build_verdict, manifest_from_files + +ROOT = Path(__file__).resolve().parents[1] +REPO = "acme/kb" +PAGE = "repos/demo/core/rules.md" +JUDGE = ModelRole("judge", "codex", "gpt-6-sol", "medium") +KEY = Ed25519PrivateKey.generate() + + +def _rule(rule_id: str, cite: str = "PR #10") -> str: + return (f"## {rule_id} — keep the demo queue bounded\n\n" + "- 触发：修改 demo queue 的容量或背压逻辑时。\n" + "- 强制：队列满时拒绝新请求并返回明确错误，不能无界增长。\n" + f"- 验收：测试覆盖满队列拒绝路径。 ^[{cite}]\n") + + +def _tree() -> dict[str, str]: + return { + "repos/demo/_index.md": "# demo\n", + "repos/demo/core/_index.md": "# core\n\n- [rules](rules.md)\n", + "repos/demo/api/_index.md": "# api\n", + "repos/demo/_routes.yaml": ( + "schema_version: 1\nowners:\n - owner: core\n path: repos/demo/core/rules.md\n" + " signals: [queue]\n scope_prefixes: [demo/core/]\n"), + PAGE: ("---\ntitle: \"Demo core rules\"\ncreated: 2026-09-01\nupdated: 2026-09-01\n" + "type: rule\ntags: [demo]\nsources: [\"PR #10\"]\n---\n\n# Demo core rules\n\n" + + _rule("DEMO-1a")), + } + + +class _Gateway: + def call_json(self, role, *, system, prompt, validate=None): + if '"directory"' in prompt: + data = {"verdict": "consistent", "conflicts": []} + else: + dims = json.loads(prompt.split("<untrusted_data>\n", 1)[1].rsplit("\n</untrusted_data>", 1)[0]) + data = {"dimensions": {d: "yes" for d in dims["dimensions_to_answer"]}, "reasons": {}} + return ModelReply(role, data, json.dumps(data), role.model, {}, 0.1) + + +# -- a git "remote" with main and PR refs, and the verifier's clone ----------------- + +def _git(cwd: Path, *args: str, env: dict | None = None) -> str: + return subprocess.run(["git", *args], cwd=cwd, check=True, capture_output=True, text=True, + env={**os.environ, **(env or {})}).stdout.strip() + + +def _write(repo: Path, files: dict[str, str | None], *, prefix: str = "knowledge/") -> None: + for rel, text in files.items(): + path = repo / (prefix + rel) + if text is None: + path.unlink() + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _commit(repo: Path, message: str) -> str: + _git(repo, "add", "-A") + _git(repo, "-c", "user.name=t", "-c", "user.email=t@e", "commit", "-q", "--allow-empty", "-m", message) + return _git(repo, "rev-parse", "HEAD") + + +class World: + """remote repo + verifier clone + a fake GitHub API.""" + + def __init__(self, tmp_path: Path): + self.remote = tmp_path / "remote" + self.remote.mkdir() + _git(self.remote, "init", "-q", "-b", "main") + _write(self.remote, _tree()) + (self.remote / "src").mkdir() + (self.remote / "src" / "app.py").write_text("x = 1\n") + self.main = _commit(self.remote, "base") + self.ws = tmp_path / "ws" + _git(tmp_path, "clone", "-q", str(self.remote), str(self.ws)) + self.prs: dict[int, dict] = {} + self.comments: dict[int, list[dict]] = {} + self.reviews: dict[tuple[int, int], dict] = {} + self.posts: list[tuple[str, dict]] = [] + self.holds = {"global": False, "repos": [], "prs": []} + self.holds_age = 0.0 + self.codeowners = "/.github/ @alice\n" + + # branches ---------------------------------------------------------------- + def branch(self, number: int, files: dict[str, str | None], *, code: dict[str, str] | None = None, + start: str | None = None) -> str: + _git(self.remote, "checkout", "-q", "-B", f"pr-{number}", start or self.main) + _write(self.remote, files) + if code: + _write(self.remote, code, prefix="") + head = _commit(self.remote, f"pr {number}") + _git(self.remote, "update-ref", f"refs/pull/{number}/head", head) + _git(self.remote, "checkout", "-q", "main") + self.prs[number] = {"number": number, "state": "open", "draft": False, "labels": [], + "head": {"sha": head}, "base": {"ref": "main"}} + return head + + def advance_main(self, files: dict[str, str | None], *, code: dict[str, str] | None = None) -> str: + _git(self.remote, "checkout", "-q", "main") + _write(self.remote, files) + if code: + _write(self.remote, code, prefix="") + self.main = _commit(self.remote, "main moves") + return self.main + + def merge_group(self, numbers: list[int]) -> str: + """gh-readonly-queue style: one merge commit per PR on top of main.""" + _git(self.remote, "checkout", "-q", "-B", "queue", self.main) + for number in numbers: + _git(self.remote, "-c", "user.name=q", "-c", "user.email=q@e", "merge", "-q", "--no-ff", + "-m", f"Merge pull request #{number}", self.prs[number]["head"]["sha"]) + head = _git(self.remote, "rev-parse", "HEAD") + _git(self.remote, "checkout", "-q", "main") + _git(self.ws, "fetch", "-q", "origin", f"+{head}:refs/queue/head") + return head + + # verdicts ---------------------------------------------------------------- + def verdict(self, number: int, head_files: dict[str, str], *, key=KEY, source="auto", pr=None, + head_sha=None, issued_at=None, mutate=None, **kw) -> dict: + base = _tree() + head = {**base, **head_files} + touched = set(head_files) + decision = run_gate(base=base, head=head, changes=changes_between(base, head), external_texts={}, + evidence=[], gateway=_Gateway(), judge=JUDGE, release="v1", repo_dir="repos/demo", + retire_ratio=1.0) + assert decision.status == "pass", decision.reasons + verdict = build_verdict( + repository=REPO, pr=pr or number, head_sha=head_sha or self.prs[number]["head"]["sha"], + context_base_sha=kw.pop("context_base_sha", self.main), + manifest=manifest_from_files({k: v for k, v in base.items() if k in touched}, + {k: v for k, v in head.items() if k in touched}), + blocks=[] if source != "auto" else signable_blocks(decision), + consistency=[] if source != "auto" else [ + {"owner_dir": c["owner_dir"], "verdict": c["verdict"], "pages": c["pages"]} + for c in decision.consistency], + release="v1", upstream={}, facts=[], models={}, source=source, + issued_at=time.time() - 5 if issued_at is None else issued_at, **kw) + if mutate: + mutate(verdict) + envelope = sign("kb-gate-verdict", verdict, key) + self.comments.setdefault(number, []).append( + {"body": f"{gv.VERDICT_MARKER}\n```json\n{json.dumps(envelope, ensure_ascii=False)}\n```\n"}) + return verdict + + # the fake GitHub API ----------------------------------------------------- + def get(self, path: str): + path = path.split("?", 1)[0] + parts = path.strip("/").split("/") + if parts[-2:] == ["comments"] or parts[-1] == "comments": + return self.comments.get(int(parts[-2]), []) + if len(parts) == 7 and parts[5] == "reviews": + return self.reviews[(int(parts[4]), int(parts[6]))] + if parts[-1] == "pulls": + return [pr for pr in self.prs.values() if pr["state"] == "open"] + if parts[-2] == "pulls": + return self.prs[int(parts[-1])] + raise AssertionError(path) + + def get_all(self, path: str, *, pages: int = 10): + return self.get(path) + + def post(self, path: str, body: dict): + self.posts.append((path, body)) + return {} + + def context(self) -> gv.Context: + payload = {"issued_at": time.time() - self.holds_age, "sequence": 1, **self.holds} + return gv.Context(git=gv.Git(self.ws), github=self, repository=REPO, public_key=KEY.public_key(), + holds_loader=lambda: sign("kb-holds", payload, KEY), + codeowners=gv.parse_codeowners(self.codeowners), now=time.time()) + + +def _add_rule() -> dict[str, str]: + base = _tree() + return apply_operations(base, [Op("add", PAGE, "DEMO-2a", _rule("DEMO-2a", "PR #11"))], + release="v1", today="2026-09-28").files + + +def _pr(world: World, ctx=None, number: int = 1) -> list[str]: + _head, problems = gv.verify_pr(ctx or world.context(), number) + return problems + + +@pytest.fixture +def world(tmp_path): + return World(tmp_path) + + +# -- PR stage ---------------------------------------------------------------------- + +def test_a_pr_that_does_not_touch_knowledge_passes_without_a_verdict(world): + world.branch(1, {}, code={"src/app.py": "x = 2\n"}) + assert _pr(world) == [] + + +def test_a_signed_auto_verdict_for_this_head_passes(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + assert _pr(world) == [] + + +def test_a_knowledge_pr_without_a_valid_verdict_fails(world): + files = _add_rule() + world.branch(1, files) + assert any("no valid signed verdict" in p for p in _pr(world)) + # replayed from another PR, bound to another head, signed by another key, expired + world.verdict(1, files, pr=2) + world.verdict(1, files, head_sha="f" * 40) + world.verdict(1, files, key=Ed25519PrivateKey.generate()) + world.verdict(1, files, issued_at=time.time() - 73 * 3600) + problems = _pr(world) + assert len(problems) == 1 and "no valid signed verdict" in problems[0] + assert "another repository or pull request" in problems[0] + + +def test_a_verdict_whose_manifest_omits_a_change_fails(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files, mutate=lambda v: v.update(manifest=[])) + assert "the signed patch manifest does not match the change as applied" in _pr(world) + + +def test_a_block_table_that_does_not_cover_the_change_fails(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files, mutate=lambda v: v.update(blocks=v["blocks"][1:])) + assert "the signed block table does not cover exactly the changed blocks" in _pr(world) + + +def test_mixed_code_and_knowledge_and_executable_files_fail_even_with_a_verdict(world): + files = _add_rule() + world.branch(1, files, code={"src/app.py": "x = 3\n"}) + world.verdict(1, files) + problems = _pr(world) + assert "the signed patch manifest does not match the change as applied" in problems + assert any(p.startswith("L1 path_not_whitelisted src/app.py") for p in problems) + + world.branch(2, {}) + _git(world.remote, "checkout", "-q", "pr-2") + page = world.remote / "knowledge" / PAGE + page.write_text(page.read_text() + "\n") + page.chmod(0o755) + head = _commit(world.remote, "exec bit") + _git(world.remote, "update-ref", "refs/pull/2/head", head) + _git(world.remote, "checkout", "-q", "main") + world.prs[2]["head"]["sha"] = head + problems = _pr(world, number=2) + assert any(p.startswith("L1 ") and "100755" in p for p in problems), problems + + +def test_holds_labels_and_a_stale_or_missing_hold_list_fail_closed(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + assert _pr(world) == [] + world.prs[1]["labels"] = [{"name": "kb:hold"}] + assert "PR #1 carries kb:hold" in _pr(world) + world.prs[1]["labels"] = [] + for holds, needle in (({"global": True}, "global pause"), ({"prs": [1]}, "hold list"), + ({"repos": ["demo"]}, "repository demo is paused")): + world.holds = {"global": False, "repos": [], "prs": [], **holds} + assert any(needle in p for p in _pr(world)), holds + world.holds = {"global": False, "repos": [], "prs": []} + world.holds_age = 11 * 60 + assert any("stale" in p for p in _pr(world)) + world.holds_age = 0 + ctx = world.context() + + def unreachable(): + raise gv.GateError("hold list unreachable: timeout") + ctx.holds_loader = unreachable + assert any("unreachable" in p for p in _pr(world, ctx)) + + +def test_a_context_change_since_judging_invalidates_the_verdict(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + world.advance_main({}, code={"src/app.py": "x = 9\n"}) # unrelated: still fine + assert _pr(world) == [] + world.advance_main({"repos/demo/core/_index.md": "# core\n\n- [rules](rules.md)\n\nmore\n"}) + problems = _pr(world) + assert any("context changed since the verdict was judged: repos/demo/core/_index.md" in p for p in problems) + + +def test_the_status_is_posted_on_the_verified_head(world): + files = _add_rule() + head = world.branch(1, files) + world.verdict(1, files) + root = world.ws + (root / ".github").mkdir() + (root / ".github" / "kb-gate.pub").write_text(public_key_text(KEY.public_key()) + "\n") + original = gv.fetch_holds + payload = {"issued_at": time.time(), "sequence": 1, "global": False, "repos": [], "prs": []} + gv.fetch_holds = lambda url: sign("kb-holds", payload, KEY) + try: + assert gv.main(["pr", "--repository", REPO, "--pr", "1", "--root", str(root), "--post-status"], + github=world) == 0 + finally: + gv.fetch_holds = original + assert world.posts == [(f"/repos/{REPO}/statuses/{head}", + {"state": "success", "context": "kb-gate", "description": "knowledge verdict verified"})] + + +# -- human-approved ------------------------------------------------------------------ + +def test_a_human_approved_verdict_needs_a_live_maintainer_approval_on_this_head(world): + files = _add_rule() + head = world.branch(1, files) + world.verdict(1, files, source="human-approved", review_ids=(7,), reviewers=("alice",)) + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + world.reviews[(1, 7)] = {"state": "APPROVED", "commit_id": head, "user": {"login": "alice"}} + assert _pr(world) == [] + world.reviews[(1, 7)]["state"] = "DISMISSED" + assert "review 7 is no longer APPROVED" in _pr(world) + world.reviews[(1, 7)]["state"] = "APPROVED" + world.prs[1]["labels"] = [] + assert "kb:human-approved was removed" in _pr(world) + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + world.codeowners = "/.github/ @bob\n" + assert any("knowledge maintainer" in p for p in _pr(world)) + + +def test_a_human_approval_admits_paths_outside_the_auto_merge_whitelist(world): + tool = {"tools/check.py": "print('ok')\n"} + head = world.branch(1, tool) + manifest = manifest_from_files({}, tool) + world.verdict(1, {}, source="human-approved", review_ids=(7,), reviewers=("alice",), + mutate=lambda v: v.update(manifest=manifest)) + world.reviews[(1, 7)] = {"state": "APPROVED", "commit_id": head, "user": {"login": "alice"}} + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + assert _pr(world) == [] + world.verdict(1, {}, mutate=lambda v: v.update(manifest=manifest, source="auto")) + world.comments[1] = world.comments[1][1:] # only the auto verdict remains + assert any(p.startswith("L1 path_not_whitelisted knowledge/tools/check.py") for p in _pr(world)) + + +def test_a_human_approved_edit_of_a_non_governed_markdown_file_passes(world): + world.advance_main({"skills/demo/SKILL.md": "# demo skill\n"}) + edit = {"skills/demo/SKILL.md": "# demo skill\n\nmore\n"} + head = world.branch(1, edit) + manifest = manifest_from_files({"skills/demo/SKILL.md": "# demo skill\n"}, edit) + world.verdict(1, {}, source="human-approved", review_ids=(7,), reviewers=("alice",), + mutate=lambda v: v.update(manifest=manifest)) + world.reviews[(1, 7)] = {"state": "APPROVED", "commit_id": head, "user": {"login": "alice"}} + world.prs[1]["labels"] = [{"name": "kb:human-approved"}] + assert _pr(world) == [] + + +def test_a_hold_on_general_applies_to_general_pages(world): + world.branch(1, {"general/guide.md": "# guide\n"}) + world.holds["repos"] = ["general"] + assert "repository general is paused by the knowledge service" in _pr(world) + + +# -- merge group ------------------------------------------------------------------- + +def test_the_merge_group_verifies_every_knowledge_segment(world): + files = _add_rule() + world.branch(1, files) + world.branch(2, {}, code={"src/app.py": "x = 5\n"}) + world.verdict(1, files) + # knowledge first, code last: the knowledge segment is still verified + assert gv.verify_merge_group(world.context(), world.merge_group([1, 2])) == [] + assert gv.verify_merge_group(world.context(), world.merge_group([2, 1])) == [] + world.comments[1] = [] + problems = gv.verify_merge_group(world.context(), world.merge_group([1, 2])) + assert problems and all(p.startswith("PR #1: ") for p in problems) + + +def test_the_merge_group_rechecks_draft_and_hold_state(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + group = world.merge_group([1]) + world.prs[1]["draft"] = True + assert "PR #1: PR #1 is a draft (paused)" in gv.verify_merge_group(world.context(), group) + world.prs[1]["draft"] = False + world.holds["prs"] = [1] + assert any("hold list" in p for p in gv.verify_merge_group(world.context(), group)) + + +def test_a_merge_group_segment_that_is_not_a_merge_commit_is_rejected(world): + files = _add_rule() + head = world.branch(1, files) + world.verdict(1, files) + _git(world.ws, "fetch", "-q", "origin", f"+{head}:refs/queue/head") # fast-forward "group" + problems = gv.verify_merge_group(world.context(), head) + assert problems == [f"segment {head[:12]} is not a two-parent merge commit; the group cannot be attributed"] + + +def test_a_later_queued_pr_in_the_same_owner_directory_invalidates_the_earlier_verdict(world): + files = _add_rule() + world.branch(1, files) + world.verdict(1, files) + other = {"repos/demo/core/_index.md": "# core\n\n- [rules](rules.md)\n\nnote\n"} + world.branch(3, other) + world.verdict(3, other) # judged on main; PR 1 lands in front of it in the group + problems = gv.verify_merge_group(world.context(), world.merge_group([1, 3])) + assert any(p.startswith("PR #3: ") and "context changed" in p for p in problems) + + +def test_a_rule_retired_in_front_of_a_pr_that_cites_it_fails_in_the_landing_tree(world): + base = _tree() + retire = apply_operations(base, [Op("retire", PAGE, "DEMO-1a", reason="upstream-removed", + evidence="PR #13")], release="v1", today="2026-09-28").files + text = _rule("DEMO-3a", "PR #12").replace("队列满时", "与 DEMO-1a 一致，队列满时") + cite = apply_operations(base, [Op("add", "repos/demo/api/rules.md", "DEMO-3a", text, + page_title="Demo api rules")], release="v1", today="2026-09-28").files + world.branch(1, retire) + world.branch(3, cite) + world.verdict(1, retire) + world.verdict(3, cite) + assert _pr(world, number=1) == [] and _pr(world, number=3) == [] # each is fine alone + for order in ([1, 3], [3, 1]): + problems = gv.verify_merge_group(world.context(), world.merge_group(order)) + assert any(p.startswith("PR #1: L1 dangling_reference repos/demo/api/rules.md DEMO-1a") + for p in problems), (order, problems) + + +# -- the pinned bundle ------------------------------------------------------------- + +def test_the_bundle_is_fresh(): + proc = subprocess.run([sys.executable, str(ROOT / "tools" / "build_kb_gate_bundle.py"), "--check"], + capture_output=True, text=True) + assert proc.returncode == 0, proc.stdout + + +def test_the_bundle_imports_only_the_stdlib_its_siblings_and_pinned_deps(): + bundle = ROOT / ".github" / "kb-gate" + siblings = {p.stem for p in bundle.glob("*.py")} + allowed = siblings | {"yaml", "cryptography"} | set(sys.stdlib_module_names) + for path in bundle.glob("*.py"): + for node in ast.walk(ast.parse(path.read_text())): + names = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ""] + for name in names: + assert name.split(".")[0] in allowed, f"{path.name} imports {name}" + lock = (bundle / "requirements.lock").read_text() + for pin in ("cryptography==", "cffi==", "pycparser==", "PyYAML=="): + assert pin in lock + assert "--hash=sha256:" in lock + + +def test_the_bundled_verifier_runs_standalone(world, tmp_path): + """Run .github/kb-gate/kb_gate.py as the workflow does, outside the source + tree: a code-only group passes, a knowledge group without the key fails.""" + world.branch(2, {}, code={"src/app.py": "x = 5\n"}) + group = world.merge_group([2]) + script = ROOT / ".github" / "kb-gate" / "kb_gate.py" + env = {"PATH": os.environ["PATH"], "HOME": str(tmp_path), "GITHUB_TOKEN": "none"} + run = lambda head: subprocess.run( # noqa: E731 + [sys.executable, "-E", "-s", str(script), "merge-group", "--repository", REPO, + "--head-sha", head, "--root", str(world.ws)], cwd=tmp_path, env=env, capture_output=True, text=True) + proc = run(group) + assert proc.returncode == 0, proc.stdout + proc.stderr + world.branch(1, _add_rule()) + proc = run(world.merge_group([1])) + assert proc.returncode == 1 and "kb-gate.pub is not committed" in proc.stdout diff --git a/tools/build_kb_gate_bundle.py b/tools/build_kb_gate_bundle.py new file mode 100644 index 00000000..71dfb73a --- /dev/null +++ b/tools/build_kb_gate_bundle.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Generate (or check) the pinned kb-gate verifier bundle in ``.github/kb-gate/``. + +The required check ``kb-gate`` runs ONLY from this bundle, never from ``src/``, +``tools/`` or ``knowledge/tools/``: weakening validation logic in a code PR does +not weaken the gate, and changing the bundle is a ``.github/**`` change that +CODEOWNERS must approve. The bundle is a verbatim copy of the verifier modules +with a generated-file header, plus ``requirements.lock`` (dependencies pinned by +hash) and ``MANIFEST.sha256`` (every bundle file's hash). + + python tools/build_kb_gate_bundle.py # regenerate the module copies + manifest + python tools/build_kb_gate_bundle.py --check # exit 1 when the bundle is stale + python tools/build_kb_gate_bundle.py --lock # re-pin requirements.lock from PyPI (network) + +A stale bundle keeps the gate at its previous strictness; ``--check`` in CI only +reminds that the source moved on. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import urllib.request +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +SOURCE = REPO / "src" / "infermatrix_copilot" / "knowledge_service" +BUNDLE = REPO / ".github" / "kb-gate" +# bundle file -> source module (the verifier imports its siblings by flat name) +MODULES = { + "kb_gate.py": "gate_verifier.py", + "l1.py": "l1.py", + "lifecycle.py": "lifecycle.py", + "ops.py": "ops.py", + "signing.py": "signing.py", + "verdict.py": "verdict.py", +} +# the pinned runtime dependencies: signing needs cryptography (-> cffi -> +# pycparser), l1/lifecycle/ops need PyYAML +PINS = {"cryptography": "50.0.1", "cffi": "2.1.1", "pycparser": "3.0", "PyYAML": "6.0.3"} +STATIC = ("README.md", "config.json", "requirements.lock") +HEADER = "# GENERATED by tools/build_kb_gate_bundle.py from {src} -- do not edit here.\n" + + +def module_text(bundle_name: str) -> str: + src = SOURCE / MODULES[bundle_name] + return HEADER.format(src=src.relative_to(REPO).as_posix()) + src.read_text(encoding="utf-8") + + +def manifest_text(files: dict[str, str]) -> str: + return "".join(f"{hashlib.sha256(files[name].encode()).hexdigest()} {name}\n" for name in sorted(files)) + + +def expected_bundle() -> dict[str, str]: + files = {name: module_text(name) for name in MODULES} + for name in STATIC: + path = BUNDLE / name + if not path.is_file(): + raise SystemExit(f"{path.relative_to(REPO)} is missing") + files[name] = path.read_text(encoding="utf-8") + files["MANIFEST.sha256"] = manifest_text(files) + return files + + +def lock_text() -> str: + lines = ["# GENERATED by tools/build_kb_gate_bundle.py --lock. Install with:", + "# pip install --require-hashes --no-deps -r .github/kb-gate/requirements.lock"] + for name, version in PINS.items(): + with urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{version}/json", timeout=30) as resp: + data = json.load(resp) + hashes = sorted({item["digests"]["sha256"] for item in data["urls"]}) + lines.append(f"{name}=={version} \\") + lines.extend(f" --hash=sha256:{digest} \\" for digest in hashes[:-1]) + lines.append(f" --hash=sha256:{hashes[-1]}") + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true") + parser.add_argument("--lock", action="store_true") + args = parser.parse_args(argv) + if args.lock: + (BUNDLE / "requirements.lock").write_text(lock_text(), encoding="utf-8") + expected = expected_bundle() + actual = {p.name: p.read_text(encoding="utf-8") for p in BUNDLE.iterdir() if p.is_file()} \ + if BUNDLE.is_dir() else {} + stale = sorted(name for name in set(expected) | set(actual) if expected.get(name) != actual.get(name)) + if args.check: + for name in stale: + print(f"kb-gate bundle is stale: .github/kb-gate/{name}") + if stale: + print("regenerate with: python tools/build_kb_gate_bundle.py (a .github/** change: CODEOWNERS review)") + return 1 if stale else 0 + BUNDLE.mkdir(parents=True, exist_ok=True) + for name, text in expected.items(): + (BUNDLE / name).write_text(text, encoding="utf-8") + for name in set(actual) - set(expected): + (BUNDLE / name).unlink() + print(f"kb-gate bundle: {len(expected)} files" + (f", updated {', '.join(stale)}" if stale else ", unchanged")) + return 0 + + +if __name__ == "__main__": + sys.exit(main())