From 2511f3447a5ffbd33b57f16faaecb442aff264f9 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Mon, 28 Sep 2026 18:53:54 +0800 Subject: [PATCH] Audit releases against a generated baseline for knowledge changes The release audit enforced every issue against the committed release_baseline.yaml, so a knowledge change made right after an upstream release failed until someone advanced the adapter baseline in a separate PR. The knowledge service also could not load the auditor from an installed wheel: it lived under tools/, which the wheel does not ship. - Move the auditor to the adapter plugin adapters/vllm_omni/release_audit.py (packaged runtime data); tools/vllm_omni_release_audit.py re-exports it, so the CLI, CI workflow, imupdate skill and existing tests are unchanged. - Add --baseline-source generated (committed stays the default): the baseline is built in memory for the audited SHA pair. Only knowledge-document issues are enforced (stale_knowledge_source, knowledge_metadata_error, owner_document_missing). Pin and inventory drift, unrouted paths, stale source-map patterns and doc pins go to a `reconciliation` list, together with committed_baseline_behind / committed_inventory_differs: that list is what the adapter-baseline companion PR must fix. - Add knowledge_service.release_audit, a repo-neutral loader for a plugin's audit_for_knowledge() entry. It refuses absolute, escaping, missing or non-.py plugin paths. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/vllm-omni-release-audit.yml | 1 + adapters/vllm_omni/release_audit.py | 861 ++++++++++++++++++ doc/architecture/SPEC/knowledge_service.md | 5 + .../SPEC/knowledge_service/release_audit.md | 20 + doc/contributing/release-maintenance.md | 14 + .../knowledge_service/release_audit.py | 145 +++ test/test_release_audit.py | 151 +++ tools/vllm_omni_release_audit.py | 764 +--------------- 8 files changed, 1217 insertions(+), 744 deletions(-) create mode 100644 adapters/vllm_omni/release_audit.py create mode 100644 doc/architecture/SPEC/knowledge_service/release_audit.md create mode 100644 src/infermatrix_copilot/knowledge_service/release_audit.py diff --git a/.github/workflows/vllm-omni-release-audit.yml b/.github/workflows/vllm-omni-release-audit.yml index 46cd48346..1f67b580a 100644 --- a/.github/workflows/vllm-omni-release-audit.yml +++ b/.github/workflows/vllm-omni-release-audit.yml @@ -11,6 +11,7 @@ on: - knowledge/repos/vllm-omni/** - tools/audit_vllm_omni_release.py - tools/vllm_omni_release_audit.py + - adapters/vllm_omni/release_audit.py schedule: - cron: "23 3 * * 1" workflow_dispatch: diff --git a/adapters/vllm_omni/release_audit.py b/adapters/vllm_omni/release_audit.py new file mode 100644 index 000000000..f2d69421b --- /dev/null +++ b/adapters/vllm_omni/release_audit.py @@ -0,0 +1,861 @@ +"""Deterministic, read-only drift audit for vLLM-Omni releases. + +This is the vllm-omni release-audit plugin. It lives in the adapter (packaged +runtime data) so the knowledge service can load it from an installed wheel; +``tools/vllm_omni_release_audit.py`` re-exports it for the CLI, CI and tests. + +Two baseline sources: + +* ``committed`` (default, the historical behaviour): compare against + ``release_baseline.yaml`` and enforce every issue, including a baseline that + has not been advanced to the audited release. +* ``generated``: build the baseline in memory for the audited SHA pair and + split issues in two. ``issues`` are knowledge-document problems (a knowledge + page cites a path the release removed, unreadable metadata, an owner + document missing) and are enforced. ``reconciliation`` is everything that is + adapter/baseline maintenance (the committed pin and inventories, unrouted + upstream paths, stale source-map patterns, stale doc pins); it is reported + but never blocks, so a knowledge change does not wait for the adapter + baseline PR. The committed-vs-generated difference is reported too. +""" + +from __future__ import annotations + +import argparse +import ast +import fnmatch +import hashlib +import json +import re +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +import yaml + + +_ADAPTER_DIR = Path(__file__).resolve().parent +_REPO_ROOT = _ADAPTER_DIR.parents[1] +BASELINE_SOURCES = ("committed", "generated") +# enforced in generated mode: the knowledge documents themselves are wrong +KNOWLEDGE_ISSUE_KINDS = frozenset({ + "stale_knowledge_source", + "knowledge_metadata_error", + "owner_document_missing", +}) +_REGISTRIES = { + "autoregressive": ( + "vllm_omni/model_executor/models/registry.py", + "_OMNI_MODELS", + True, + ), + "diffusion": ( + "vllm_omni/diffusion/registry.py", + "_DIFFUSION_MODELS", + True, + ), + "pipelines": ( + "vllm_omni/config/pipeline_registry.py", + "OMNI_PIPELINES", + False, + ), +} +_UPSTREAM_SOURCE_PREFIXES = ( + ".buildkite/", + "benchmarks/", + "docs/", + "tests/", + "vllm_omni/", +) +_SOURCE_EXCLUDED_PARTS = {"_archive", "history", "incidents", "results"} +_PIN_RE = re.compile( + r"\b(?:main|v\d[0-9A-Za-z.-]*)\s*@\s*`?([0-9a-f]{8,40})", + re.IGNORECASE, +) + + +class ReleaseAuditError(RuntimeError): + """The audit could not read or understand one of its declared inputs.""" + + +@dataclass(frozen=True) +class ChangedPath: + status: str + path: str + old_path: str | None = None + + def as_dict(self) -> dict[str, str]: + item = {"status": self.status, "path": self.path} + if self.old_path: + item["old_path"] = self.old_path + return item + + +@dataclass(frozen=True) +class ReleaseAuditReport: + data: dict[str, Any] + + @property + def has_drift(self) -> bool: + return bool(self.data["issues"]) + + def to_json(self) -> str: + return ( + json.dumps( + self.data, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + +class GitSnapshotReader: + """Read immutable Git objects without checking out or importing upstream.""" + + def __init__(self, repo: Path | str): + self.repo = Path(repo).resolve() + if not self.repo.is_dir(): + raise ReleaseAuditError(f"upstream checkout does not exist: {self.repo}") + self._run_text("rev-parse", "--git-dir") + + def _run( + self, + *args: str, + text: bool, + check: bool = True, + ) -> subprocess.CompletedProcess: + try: + result = subprocess.run( + ["git", "-C", str(self.repo), *args], + capture_output=True, + text=text, + encoding="utf-8" if text else None, + errors="replace" if text else None, + timeout=60, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ReleaseAuditError(f"git {' '.join(args)} failed: {exc}") from exc + if check and result.returncode: + stderr = ( + result.stderr.strip() + if text + else result.stderr.decode("utf-8", errors="replace").strip() + ) + raise ReleaseAuditError( + f"git {' '.join(args)} failed ({result.returncode}): {stderr}" + ) + return result + + def _run_text(self, *args: str, check: bool = True) -> str: + return self._run(*args, text=True, check=check).stdout.strip() + + def resolve(self, ref: str) -> str: + sha = self._run_text("rev-parse", "--verify", f"{ref}^{{commit}}") + if not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ReleaseAuditError(f"ref did not resolve to a commit: {ref}") + return sha + + def read_text(self, commit: str, path: str) -> str: + return self._run_text("show", f"{commit}:{path}") + + def tree_files(self, commit: str) -> list[str]: + raw = self._run( + "ls-tree", + "-r", + "--name-only", + "-z", + commit, + text=False, + ).stdout + return sorted( + part.decode("utf-8", errors="surrogateescape") + for part in raw.split(b"\0") + if part + ) + + def changed_paths(self, old: str, new: str) -> list[ChangedPath]: + raw = self._run( + "diff", + "--name-status", + "-z", + "-M", + old, + new, + "--", + text=False, + ).stdout + fields = [ + part.decode("utf-8", errors="surrogateescape") + for part in raw.split(b"\0") + if part + ] + changed: list[ChangedPath] = [] + index = 0 + while index < len(fields): + status = fields[index] + index += 1 + kind = status[:1] + if kind in {"R", "C"}: + if index + 1 >= len(fields): + raise ReleaseAuditError("unexpected truncated git rename output") + old_path, new_path = fields[index], fields[index + 1] + index += 2 + changed.append( + ChangedPath(status=status, path=new_path, old_path=old_path) + ) + else: + if index >= len(fields): + raise ReleaseAuditError("unexpected truncated git diff output") + changed.append(ChangedPath(status=status, path=fields[index])) + index += 1 + return sorted(changed, key=lambda item: (item.path, item.old_path or "")) + + +def _assigned_dict(source: str, variable: str, path: str) -> ast.Dict: + try: + tree = ast.parse(source, filename=path) + except SyntaxError as exc: + raise ReleaseAuditError(f"cannot parse {path}: {exc}") from exc + for node in tree.body: + value: ast.AST | None = None + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == variable + for target in node.targets + ): + value = node.value + elif ( + isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == variable + ): + value = node.value + if value is not None: + if not isinstance(value, ast.Dict): + raise ReleaseAuditError(f"{path}:{variable} is not a dict literal") + return value + raise ReleaseAuditError(f"{path} does not define {variable}") + + +def _registry_entries( + source: str, + variable: str, + path: str, + include_targets: bool, +) -> dict[str, list[str]] | list[str]: + dictionary = _assigned_dict(source, variable, path) + entries: dict[str, list[str]] = {} + keys: list[str] = [] + for key_node, value_node in zip(dictionary.keys, dictionary.values): + if not isinstance(key_node, ast.Constant) or not isinstance( + key_node.value, str + ): + raise ReleaseAuditError(f"{path}:{variable} has a non-string key") + key = key_node.value + if key in keys: + raise ReleaseAuditError(f"{path}:{variable} has duplicate key {key!r}") + keys.append(key) + if include_targets: + try: + value = ast.literal_eval(value_node) + except (ValueError, TypeError, SyntaxError) as exc: + raise ReleaseAuditError( + f"{path}:{variable}[{key!r}] is not a literal target tuple" + ) from exc + if ( + not isinstance(value, tuple) + or len(value) != 3 + or not all(isinstance(part, str) for part in value) + ): + raise ReleaseAuditError( + f"{path}:{variable}[{key!r}] is not a 3-string target tuple" + ) + entries[key] = list(value) + return dict(sorted(entries.items())) if include_targets else sorted(keys) + + +def _fingerprint(value: Any) -> dict[str, int | str]: + encoded = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return { + "count": len(value), + "sha256": hashlib.sha256(encoded).hexdigest(), + } + + +def snapshot_inventory(reader: GitSnapshotReader, commit: str) -> dict[str, Any]: + inventories: dict[str, Any] = {} + for name, (path, variable, include_targets) in _REGISTRIES.items(): + inventories[name] = _registry_entries( + reader.read_text(commit, path), + variable, + path, + include_targets, + ) + inventories["deploy_yamls"] = sorted( + path + for path in reader.tree_files(commit) + if path.startswith("vllm_omni/deploy/") + and path.casefold().endswith((".yaml", ".yml")) + ) + return inventories + + +def _inventory_delta(old: Any, new: Any) -> dict[str, Any]: + if isinstance(old, dict) and isinstance(new, dict): + old_keys, new_keys = set(old), set(new) + return { + "added": sorted(new_keys - old_keys), + "removed": sorted(old_keys - new_keys), + "changed": sorted( + key for key in old_keys & new_keys if old[key] != new[key] + ), + } + old_set, new_set = set(old), set(new) + return { + "added": sorted(new_set - old_set), + "removed": sorted(old_set - new_set), + "changed": [], + } + + +def _load_yaml_mapping(path: Path, label: str) -> dict[str, Any]: + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + raise ReleaseAuditError(f"cannot read {label} {path}: {exc}") from exc + if not isinstance(value, dict): + raise ReleaseAuditError(f"{label} must be a YAML mapping: {path}") + return value + + +def _matches(path: str, pattern: str) -> bool: + normalized = pattern.replace("\\", "/") + if normalized.endswith("/"): + return path.startswith(normalized) + return fnmatch.fnmatchcase(path, normalized) + + +def _route_path(path: str, baseline: dict[str, Any]) -> tuple[list[str], list[str]]: + owners = sorted( + owner + for owner, patterns in (baseline.get("path_owners") or {}).items() + if any(_matches(path, str(pattern)) for pattern in patterns or []) + ) + ignored = sorted( + str(item.get("reason") or "no reason supplied") + for item in baseline.get("ignored_paths") or [] + if isinstance(item, dict) and _matches(path, str(item.get("pattern") or "")) + ) + return owners, ignored + + +def _adapter_modules(path: str, manifest: dict[str, Any]) -> list[str]: + modules: list[str] = [] + for module, spec in (manifest.get("modules") or {}).items(): + for pattern in (spec or {}).get("local_paths") or []: + prefix = str(pattern).replace("\\", "/").rstrip("*").rstrip("/") + if path.startswith(prefix): + modules.append(str(module)) + break + return sorted(modules) + + +def _source_path(value: str) -> str | None: + candidate = value.strip().strip("`") + if not candidate.startswith(_UPSTREAM_SOURCE_PREFIXES): + return None + candidate = candidate.split("::", 1)[0] + candidate = re.sub(r":\d+(?:-\d+)?$", "", candidate) + return candidate + + +def _path_exists(files: Iterable[str], source_path: str) -> bool: + file_list = files if isinstance(files, list) else list(files) + if any(char in source_path for char in "*?["): + return any(fnmatch.fnmatchcase(path, source_path) for path in file_list) + if source_path.endswith("/"): + return any(path.startswith(source_path) for path in file_list) + return source_path in file_list or any( + path.startswith(source_path.rstrip("/") + "/") for path in file_list + ) + + +def _knowledge_source_issues( + knowledge_root: Path, + old_files: list[str], + new_files: list[str], + changes: list[ChangedPath], +) -> list[dict[str, str]]: + repo_root = knowledge_root / "repos" / "vllm-omni" + if not repo_root.is_dir(): + raise ReleaseAuditError(f"vLLM-Omni knowledge root is missing: {repo_root}") + rename_map = { + item.old_path: item.path + for item in changes + if item.old_path and item.status.startswith("R") + } + issues: list[dict[str, str]] = [] + for page in sorted(repo_root.rglob("*.md")): + if any( + part in _SOURCE_EXCLUDED_PARTS for part in page.relative_to(repo_root).parts + ): + continue + text = page.read_text(encoding="utf-8") + if not text.startswith("---"): + continue + parts = text.split("---", 2) + if len(parts) != 3: + continue + try: + metadata = yaml.safe_load(parts[1]) or {} + except yaml.YAMLError as exc: + issues.append( + { + "kind": "knowledge_metadata_error", + "document": page.relative_to(knowledge_root.parent).as_posix(), + "detail": str(exc).splitlines()[0], + } + ) + continue + sources = metadata.get("sources") or [] + if not isinstance(sources, list): + issues.append( + { + "kind": "knowledge_metadata_error", + "document": page.relative_to(knowledge_root.parent).as_posix(), + "detail": "sources must be a list", + } + ) + continue + for raw_source in sources: + if not isinstance(raw_source, str): + continue + source_path = _source_path(raw_source) + if not source_path: + continue + if _path_exists(old_files, source_path) and not _path_exists( + new_files, source_path + ): + issue = { + "kind": "stale_knowledge_source", + "document": page.relative_to(knowledge_root.parent).as_posix(), + "source": source_path, + } + if source_path in rename_map: + issue["renamed_to"] = rename_map[source_path] + issues.append(issue) + return sorted( + issues, + key=lambda item: ( + item["kind"], + item.get("document", ""), + item.get("source", ""), + ), + ) + + +def _pin_issues( + project_root: Path, + baseline: dict[str, Any], + audited_sha: str, +) -> list[dict[str, str]]: + issues: list[dict[str, str]] = [] + for rel in baseline.get("pin_documents") or []: + document = project_root / str(rel) + if not document.is_file(): + issues.append( + { + "kind": "pin_document_missing", + "document": str(rel), + } + ) + continue + pins = sorted(set(_PIN_RE.findall(document.read_text(encoding="utf-8")))) + if not pins: + issues.append( + { + "kind": "pin_marker_missing", + "document": str(rel), + } + ) + continue + for pin in pins: + if not audited_sha.startswith(pin): + issues.append( + { + "kind": "stale_pin", + "document": str(rel), + "actual": pin, + "expected": audited_sha, + } + ) + return issues + + +def _policy_issues( + project_root: Path, + baseline: dict[str, Any], + target_files: list[str], +) -> list[dict[str, str]]: + issues: list[dict[str, str]] = [] + owner_documents = baseline.get("owner_documents") or {} + for owner, patterns in sorted((baseline.get("path_owners") or {}).items()): + documents = owner_documents.get(owner) or [] + if not documents: + issues.append({"kind": "owner_document_missing", "owner": str(owner)}) + for document in documents: + if not (project_root / str(document)).is_file(): + issues.append( + { + "kind": "owner_document_missing", + "owner": str(owner), + "document": str(document), + } + ) + for pattern in patterns or []: + if not _path_exists(target_files, str(pattern)): + issues.append( + { + "kind": "stale_source_map_path", + "owner": str(owner), + "path": str(pattern), + } + ) + return issues + + +def audit_release( + *, + upstream_repo: Path | str, + from_ref: str, + to_ref: str, + baseline_path: Path | str, + adapter_manifest_path: Path | str, + knowledge_root: Path | str, + project_root: Path | str = _REPO_ROOT, + baseline_source: str = "committed", +) -> ReleaseAuditReport: + if baseline_source not in BASELINE_SOURCES: + raise ReleaseAuditError(f"baseline_source must be one of {BASELINE_SOURCES}") + reader = GitSnapshotReader(upstream_repo) + old_sha, new_sha = reader.resolve(from_ref), reader.resolve(to_ref) + baseline_path = Path(baseline_path) + committed = _load_yaml_mapping(baseline_path, "release baseline") + manifest = _load_yaml_mapping(Path(adapter_manifest_path), "adapter manifest") + if committed.get("schema_version") != 1: + raise ReleaseAuditError("release baseline schema_version must be 1") + + old_inventory = snapshot_inventory(reader, old_sha) + new_inventory = snapshot_inventory(reader, new_sha) + baseline = committed + if baseline_source == "generated": + baseline = { + **committed, + "upstream": { + **(committed.get("upstream") or {}), + "audited_sha": new_sha, + "previous_audited_sha": old_sha, + }, + "inventories": { + name: _fingerprint(value) for name, value in new_inventory.items() + }, + } + changes = reader.changed_paths(old_sha, new_sha) + old_files, new_files = reader.tree_files(old_sha), reader.tree_files(new_sha) + + issues: list[dict[str, Any]] = [] + expected_sha = str((baseline.get("upstream") or {}).get("audited_sha") or "") + if expected_sha != new_sha: + issues.append( + { + "kind": "baseline_pin_mismatch", + "baseline": expected_sha, + "target": new_sha, + } + ) + expected_old_sha = str( + (baseline.get("upstream") or {}).get("previous_audited_sha") or "" + ) + if expected_sha == new_sha and expected_old_sha != old_sha: + issues.append( + { + "kind": "baseline_from_mismatch", + "baseline": expected_old_sha, + "target": old_sha, + } + ) + + actual_fingerprints = { + name: _fingerprint(value) for name, value in new_inventory.items() + } + expected_fingerprints = baseline.get("inventories") or {} + for name in sorted(actual_fingerprints): + if expected_fingerprints.get(name) != actual_fingerprints[name]: + issues.append( + { + "kind": "inventory_mismatch", + "inventory": name, + "baseline": expected_fingerprints.get(name), + "target": actual_fingerprints[name], + } + ) + + routing: list[dict[str, Any]] = [] + adapter_uncovered: list[str] = [] + for change in changes: + owners, ignored = _route_path(change.path, baseline) + adapter_modules = _adapter_modules(change.path, manifest) + route = { + **change.as_dict(), + "owners": owners, + "ignored": ignored, + "adapter_modules": adapter_modules, + "knowledge_documents": sorted( + { + str(document) + for owner in owners + for document in (baseline.get("owner_documents") or {}).get( + owner, [] + ) + } + ), + } + routing.append(route) + if change.path.startswith("vllm_omni/") and not adapter_modules: + adapter_uncovered.append(change.path) + if not owners and not ignored: + issues.append({"kind": "unmatched_path", "path": change.path}) + elif len(owners) > 1 or (owners and ignored): + issues.append( + { + "kind": "suspicious_path_route", + "path": change.path, + "owners": owners, + "ignored": ignored, + } + ) + + issues.extend( + _knowledge_source_issues( + Path(knowledge_root), + old_files, + new_files, + changes, + ) + ) + issues.extend(_pin_issues(Path(project_root), baseline, new_sha)) + issues.extend( + _policy_issues( + Path(project_root), + baseline, + new_files, + ) + ) + def _order(item: dict[str, Any]) -> tuple[str, str, str, str]: + return ( + str(item.get("kind", "")), + str(item.get("path", "")), + str(item.get("document", "")), + str(item.get("inventory", "")), + ) + + reconciliation: list[dict[str, Any]] = [] + if baseline_source == "generated": + reconciliation = [i for i in issues if i["kind"] not in KNOWLEDGE_ISSUE_KINDS] + issues = [i for i in issues if i["kind"] in KNOWLEDGE_ISSUE_KINDS] + committed_upstream = committed.get("upstream") or {} + if str(committed_upstream.get("audited_sha") or "") != new_sha: + reconciliation.append({ + "kind": "committed_baseline_behind", + "baseline": str(committed_upstream.get("audited_sha") or ""), + "target": new_sha, + }) + elif str(committed_upstream.get("previous_audited_sha") or "") != old_sha: + # committed mode's baseline_from_mismatch, reported instead of enforced + reconciliation.append({ + "kind": "committed_previous_differs", + "baseline": str(committed_upstream.get("previous_audited_sha") or ""), + "target": old_sha, + }) + for name, fingerprint in sorted(baseline["inventories"].items()): + if (committed.get("inventories") or {}).get(name) != fingerprint: + reconciliation.append({ + "kind": "committed_inventory_differs", + "inventory": name, + "baseline": (committed.get("inventories") or {}).get(name), + "target": fingerprint, + }) + issues = sorted(issues, key=_order) + reconciliation = sorted(reconciliation, key=_order) + + deltas = { + name: _inventory_delta(old_inventory[name], new_inventory[name]) + for name in sorted(new_inventory) + } + status_counts: dict[str, int] = {} + for change in changes: + kind = change.status[:1] + status_counts[kind] = status_counts.get(kind, 0) + 1 + + data = { + "schema_version": 1, + "upstream": { + "repository": str((baseline.get("upstream") or {}).get("repository") or ""), + "from": {"ref": from_ref, "sha": old_sha}, + "to": {"ref": to_ref, "sha": new_sha}, + }, + "inventory": { + "deltas": deltas, + "target_fingerprints": actual_fingerprints, + }, + "paths": { + "counts": dict(sorted(status_counts.items())), + "changes": [item.as_dict() for item in changes], + "routing": routing, + "adapter_uncovered": sorted(set(adapter_uncovered)), + }, + "baseline_source": baseline_source, + "issues": issues, + "result": "drift" if issues else "clean", + } + if baseline_source == "generated": + data["reconciliation"] = reconciliation + data["generated_baseline"] = { + "upstream": baseline["upstream"], + "inventories": baseline["inventories"], + } + return ReleaseAuditReport(data) + + +def audit_for_knowledge( + *, + upstream_repo: Path | str, + from_ref: str, + to_ref: str, + knowledge_root: Path | str, + project_root: Path | str, +) -> dict[str, Any]: + """Knowledge-service plugin entry (repo-neutral ``ReleaseAuditor`` contract): + generated-baseline audit using this adapter's baseline and manifest.""" + return audit_release( + upstream_repo=upstream_repo, + from_ref=from_ref, + to_ref=to_ref, + baseline_path=_ADAPTER_DIR / "release_baseline.yaml", + adapter_manifest_path=_ADAPTER_DIR / "manifest.yaml", + knowledge_root=knowledge_root, + project_root=project_root, + baseline_source="generated", + ).data + + +def render_summary(report: ReleaseAuditReport) -> str: + data = report.data + old = data["upstream"]["from"]["sha"][:8] + new = data["upstream"]["to"]["sha"][:8] + deltas = data["inventory"]["deltas"] + lines = [f"vLLM-Omni release audit: {old} -> {new}"] + for name in ("autoregressive", "diffusion", "pipelines", "deploy_yamls"): + delta = deltas[name] + lines.append( + f" {name}: +{len(delta['added'])} -{len(delta['removed'])} " + f"~{len(delta['changed'])}" + ) + lines.append( + f" changed paths: {len(data['paths']['changes'])}; " + "runtime-manifest gaps (reported separately): " + f"{len(data['paths']['adapter_uncovered'])}" + ) + lines.append(f" issues: {len(data['issues'])}") + if "reconciliation" in data: + lines.append( + f" reconciliation (reported, not enforced): {len(data['reconciliation'])}") + for issue in data["issues"][:10]: + subject = ( + issue.get("path") or issue.get("document") or issue.get("inventory") or "" + ) + lines.append(f" - {issue['kind']}: {subject}".rstrip()) + if len(data["issues"]) > 10: + lines.append(f" ... {len(data['issues']) - 10} more in JSON report") + lines.append(f"RESULT: {data['result'].upper()}") + return "\n".join(lines) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Audit vLLM-Omni release drift without modifying either repo." + ) + parser.add_argument("--from", dest="from_ref", required=True) + parser.add_argument("--to", dest="to_ref", required=True) + parser.add_argument("--repo", required=True, type=Path) + parser.add_argument( + "--baseline", + type=Path, + default=_ADAPTER_DIR / "release_baseline.yaml", + ) + parser.add_argument( + "--adapter-manifest", + type=Path, + default=_ADAPTER_DIR / "manifest.yaml", + ) + parser.add_argument( + "--baseline-source", + choices=BASELINE_SOURCES, + default="committed", + help="generated: audit against an in-memory baseline for --from/--to; " + "only knowledge-document issues are enforced", + ) + parser.add_argument( + "--knowledge-root", + type=Path, + default=_REPO_ROOT / "knowledge", + ) + parser.add_argument( + "--project-root", + type=Path, + default=_REPO_ROOT, + help="root used to resolve pin_documents from the baseline", + ) + parser.add_argument("--json-output", type=Path) + parser.add_argument( + "--mode", + choices=("enforce", "report-only"), + default="enforce", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + report = audit_release( + upstream_repo=args.repo, + from_ref=args.from_ref, + to_ref=args.to_ref, + baseline_path=args.baseline, + adapter_manifest_path=args.adapter_manifest, + knowledge_root=args.knowledge_root, + project_root=args.project_root, + baseline_source=args.baseline_source, + ) + except ReleaseAuditError as exc: + print(f"release audit error: {exc}") + return 2 + print(render_summary(report)) + if args.json_output: + args.json_output.parent.mkdir(parents=True, exist_ok=True) + args.json_output.write_text(report.to_json(), encoding="utf-8") + print(f"JSON report: {args.json_output}") + return 1 if report.has_drift and args.mode == "enforce" else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/doc/architecture/SPEC/knowledge_service.md b/doc/architecture/SPEC/knowledge_service.md index f2723dd02..c390ee08c 100644 --- a/doc/architecture/SPEC/knowledge_service.md +++ b/doc/architecture/SPEC/knowledge_service.md @@ -25,3 +25,8 @@ Knowledge Ops API 2.0 sits beside the v1 curator and does not change it: deterministic half of the quality gate (`check_tree`, `check_changeset`). The three modules need only the standard library and PyYAML and import nothing else from the package, so the kb-gate verifier bundle can vendor them. + +`release_audit` loads a repository's release-audit plugin from its adapter +directory and runs it with a baseline generated for the audited SHA pair: +knowledge-document issues are enforced, adapter-baseline maintenance is +reported as reconciliation. diff --git a/doc/architecture/SPEC/knowledge_service/release_audit.md b/doc/architecture/SPEC/knowledge_service/release_audit.md new file mode 100644 index 000000000..777b1d6ce --- /dev/null +++ b/doc/architecture/SPEC/knowledge_service/release_audit.md @@ -0,0 +1,20 @@ +# knowledge_service/release_audit.py —— 规范 + + + +`LOC ~120 · 仓库中立的发版审计插件契约 · refactor-status: stable` + +## 职责 +加载仓库 adapter 目录下的发版审计插件（随 wheel 打包的运行时数据）并调用其 +`audit_for_knowledge(*, upstream_repo, from_ref, to_ref, knowledge_root, project_root)`， +返回 `ReleaseAuditResult`（`issues` 强制、`reconciliation` 仅报告、`generated_baseline`、 +审计的 SHA 对）。插件以 generated-baseline 模式运行，知识变更不必等待 adapter 基线推进。 + +## 不变量 +- 插件路径必须是 adapter 目录内的 `.py` 相对路径；绝对路径、`..`、逃逸与缺失均抛 + `ReleaseAuditPluginError`；缺少入口函数或返回格式不符同样报错。 +- 模块中不出现任何仓库名；没有插件的仓库由 L1 的通用引用存在性检查兜底。 + +## 测试 +`test_release_audit.py`（generated 模式不因未推进的已提交基线阻塞、知识修复后 clean、 +未路由路径进入 reconciliation、CLI 两种模式、插件加载与路径拒绝）。 diff --git a/doc/contributing/release-maintenance.md b/doc/contributing/release-maintenance.md index decd6aa19..feefcf9a1 100644 --- a/doc/contributing/release-maintenance.md +++ b/doc/contributing/release-maintenance.md @@ -38,6 +38,20 @@ python tools/audit_vllm_omni_release.py ` --json-output $env:TEMP\vllm-omni-release-audit.json ``` +实现位于 adapter 插件 `adapters/vllm_omni/release_audit.py`（随 wheel 打包，知识服务经 +`knowledge_service.release_audit` 加载），`tools/vllm_omni_release_audit.py` 只是转发入口。 + +两种基线来源： + +- `--baseline-source committed`（默认）：对照已提交的 `release_baseline.yaml`，全部问题 + 均强制（下文的发版流程用这一种）。 +- `--baseline-source generated`：为 `--from/--to` 这对 SHA 在内存中生成基线，只强制 + **知识文档本身**的问题（`stale_knowledge_source`、`knowledge_metadata_error`、 + `owner_document_missing`）；基线 pin/inventory、未路由路径、过期 source map 与文档 pin + 等 adapter 维护项放入 `reconciliation` 报告，不阻塞。知识服务的巡检与知识 PR 的门禁使用 + 这一种，因此知识变更不必等待 adapter 基线 PR 合并；`reconciliation` 就是该基线 PR 要 + 修的清单。 + 审计覆盖： - AR、Diffusion 和 pipeline registry； diff --git a/src/infermatrix_copilot/knowledge_service/release_audit.py b/src/infermatrix_copilot/knowledge_service/release_audit.py new file mode 100644 index 000000000..f923a050d --- /dev/null +++ b/src/infermatrix_copilot/knowledge_service/release_audit.py @@ -0,0 +1,145 @@ +"""Repo-neutral release-audit plugin contract for the knowledge service. + +A repository whose releases can invalidate knowledge ships an audit plugin in +its adapter directory (packaged runtime data, so it loads from an installed +wheel). The plugin is a Python file exposing:: + + def audit_for_knowledge(*, upstream_repo, from_ref, to_ref, + knowledge_root, project_root) -> dict + +returning at least ``issues`` (knowledge-document problems, enforced), +``reconciliation`` (adapter/baseline maintenance, reported only), +``generated_baseline`` and ``upstream.from/to.sha``. The audit runs against a +baseline generated for exactly the audited SHA pair, so a knowledge change never +waits for the committed adapter baseline to be advanced; the reconciliation list +is what the adapter-baseline companion PR must fix. + +Repositories without a plugin fall back to the generic reference-existence +checks in L1; nothing here names a repository. +""" + +from __future__ import annotations + +import importlib.util +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import Any, Callable + +ENTRY_POINT = "audit_for_knowledge" + + +class ReleaseAuditPluginError(RuntimeError): + """The configured plugin is missing, escapes its adapter, or misbehaves.""" + + +@dataclass(frozen=True) +class ReleaseAuditResult: + from_sha: str + to_sha: str + issues: tuple[dict, ...] + reconciliation: tuple[dict, ...] + generated_baseline: dict = field(default_factory=dict) + data: dict = field(default_factory=dict, compare=False) + + @property + def enforced_clean(self) -> bool: + return not self.issues + + +def load_release_auditor(adapter_dir: str | Path, module: str) -> Callable[..., dict]: + """Load ``module`` (a path relative to ``adapter_dir``) and return its entry.""" + adapter_dir = Path(adapter_dir).resolve() + pure = PurePosixPath(str(module).replace("\\", "/")) + if not str(module) or pure.is_absolute() or ".." in pure.parts or pure.suffix != ".py": + raise ReleaseAuditPluginError(f"audit plugin must be a .py path inside the adapter: {module!r}") + path = (adapter_dir / pure).resolve() + try: + path.relative_to(adapter_dir) + except ValueError as exc: + raise ReleaseAuditPluginError(f"audit plugin escapes its adapter: {module!r}") from exc + if not path.is_file(): + raise ReleaseAuditPluginError(f"audit plugin is missing: {path}") + name = f"infermatrix_release_audit_{abs(hash(str(path)))}" + loaded = sys.modules.get(name) + if loaded is None: + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise ReleaseAuditPluginError(f"audit plugin cannot be loaded: {path}") + loaded = importlib.util.module_from_spec(spec) + sys.modules[name] = loaded + try: + spec.loader.exec_module(loaded) + except Exception: + sys.modules.pop(name, None) + raise + entry = getattr(loaded, ENTRY_POINT, None) + if not callable(entry): + raise ReleaseAuditPluginError(f"audit plugin has no {ENTRY_POINT}(): {path}") + return entry + + +def run_release_audit( + adapter_dir: str | Path, + module: str, + *, + upstream_repo: str | Path, + from_ref: str, + to_ref: str, + knowledge_root: str | Path, + project_root: str | Path, +) -> ReleaseAuditResult: + entry = load_release_auditor(adapter_dir, module) + data: dict[str, Any] = entry( + upstream_repo=upstream_repo, + from_ref=from_ref, + to_ref=to_ref, + knowledge_root=knowledge_root, + project_root=project_root, + ) + return _result(data) + + +_SHA = re.compile(r"[0-9a-f]{40}") + + +def _result(data: object) -> ReleaseAuditResult: + """Validate the plugin's report strictly: a malformed report must never + read as a clean audit.""" + def bad(reason: str) -> ReleaseAuditPluginError: + return ReleaseAuditPluginError(f"audit plugin returned a malformed report: {reason}") + + if not isinstance(data, dict): + raise bad("report must be a mapping") + upstream = data.get("upstream") + if not isinstance(upstream, dict): + raise bad("upstream must be a mapping") + shas = [] + for end in ("from", "to"): + item = upstream.get(end) + sha = item.get("sha") if isinstance(item, dict) else None + if not isinstance(sha, str) or not _SHA.fullmatch(sha): + raise bad(f"upstream.{end}.sha must be a 40-character hex SHA") + shas.append(sha) + lists = {} + for key in ("issues", "reconciliation"): + value = data.get(key) + if not isinstance(value, list) or any( + not isinstance(item, dict) or not isinstance(item.get("kind"), str) or not item["kind"] + for item in value + ): + raise bad(f"{key} must be a list of {{kind, ...}} mappings") + lists[key] = tuple(value) + baseline = data.get("generated_baseline") + if not isinstance(baseline, dict) or not isinstance(baseline.get("upstream"), dict) \ + or baseline["upstream"].get("audited_sha") != shas[1]: + raise bad("generated_baseline.upstream.audited_sha must equal upstream.to.sha") + return ReleaseAuditResult( + from_sha=shas[0], + to_sha=shas[1], + issues=lists["issues"], + reconciliation=lists["reconciliation"], + generated_baseline=dict(baseline), + data=data, + ) diff --git a/test/test_release_audit.py b/test/test_release_audit.py index baba4692e..5cf00f269 100644 --- a/test/test_release_audit.py +++ b/test/test_release_audit.py @@ -353,3 +353,154 @@ def test_cli_modes_and_json_output(tmp_path): assert main(common + ["--mode", "enforce"]) == 1 assert main(common + ["--mode", "report-only"]) == 0 assert json.loads(output.read_text(encoding="utf-8"))["result"] == "drift" + + +def _generated(fixture, **overrides): + kwargs = dict( + upstream_repo=fixture["upstream"], + from_ref=str(fixture["old"]), + to_ref=str(fixture["new"]), + baseline_path=fixture["baseline"], + adapter_manifest_path=fixture["manifest"], + knowledge_root=fixture["knowledge"], + project_root=fixture["project"], + baseline_source="generated", + ) + kwargs.update(overrides) + return audit_release(**kwargs) + + +def _stale_baseline(fixture) -> None: + """The committed baseline still points at the previous release.""" + path = fixture["baseline"] + data = yaml.safe_load(path.read_text(encoding="utf-8")) + data["upstream"]["audited_sha"] = str(fixture["old"]) + data["upstream"]["previous_audited_sha"] = "0" * 40 + data["inventories"] = {name: {"count": 0, "sha256": "0"} for name in data["inventories"]} + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + + +def test_generated_baseline_does_not_block_on_an_unadvanced_committed_baseline(tmp_path): + fixture = _fixture(tmp_path) + _stale_baseline(fixture) + committed = _audit(fixture) + kinds = {i["kind"] for i in committed.data["issues"]} + assert {"baseline_pin_mismatch", "inventory_mismatch"} <= kinds + + generated = _generated(fixture) + enforced = {i["kind"] for i in generated.data["issues"]} + reconciliation = {i["kind"] for i in generated.data["reconciliation"]} + assert enforced == {"stale_knowledge_source"} # the knowledge page cites removed paths + assert {"committed_baseline_behind", "committed_inventory_differs"} <= reconciliation + assert not ({"baseline_pin_mismatch", "baseline_from_mismatch", "inventory_mismatch"} & enforced) + assert generated.data["generated_baseline"]["upstream"]["audited_sha"] == fixture["new"] + assert generated.data["baseline_source"] == "generated" + + +def test_generated_baseline_is_clean_once_knowledge_is_fixed(tmp_path): + fixture = _fixture(tmp_path) + _stale_baseline(fixture) + page = fixture["knowledge"] / "repos" / "vllm-omni" / "components" / "owner" / "rules.md" + page.write_text( + "---\ntitle: \"owner\"\nsources: [vllm_omni/deploy/renamed.yaml]\n---\n\n# Owner\n", + encoding="utf-8", + ) + generated = _generated(fixture) + assert generated.data["issues"] == [] + assert generated.data["result"] == "clean" + assert generated.data["reconciliation"] # the adapter baseline PR is still owed + # unrouted upstream paths are baseline maintenance, reported not enforced + upstream = fixture["upstream"] + (upstream / "unowned").mkdir() + (upstream / "unowned" / "f.txt").write_text("x\n", encoding="utf-8") + fixture["new"] = _commit(upstream, "unowned") + generated = _generated(fixture) + assert generated.data["issues"] == [] + assert {"kind": "unmatched_path", "path": "unowned/f.txt"} in generated.data["reconciliation"] + + +def test_cli_generated_mode_exits_zero_on_reconciliation_only(tmp_path): + fixture = _fixture(tmp_path) + _stale_baseline(fixture) + page = fixture["knowledge"] / "repos" / "vllm-omni" / "components" / "owner" / "rules.md" + page.write_text("---\ntitle: \"owner\"\nsources: []\n---\n\n# Owner\n", encoding="utf-8") + argv = [ + "--from", str(fixture["old"]), "--to", str(fixture["new"]), + "--repo", str(fixture["upstream"]), "--baseline", str(fixture["baseline"]), + "--adapter-manifest", str(fixture["manifest"]), + "--knowledge-root", str(fixture["knowledge"]), "--project-root", str(fixture["project"]), + "--mode", "enforce", + ] + assert main(argv) == 1 # committed: the stale baseline blocks + assert main(argv + ["--baseline-source", "generated"]) == 0 + + +def test_repo_neutral_plugin_loader(tmp_path): + from infermatrix_copilot.knowledge_service.release_audit import ( + ReleaseAuditPluginError, load_release_auditor, run_release_audit, + ) + + fixture = _fixture(tmp_path) + adapter = Path(__file__).resolve().parents[1] / "adapters" / "vllm_omni" + entry = load_release_auditor(adapter, "release_audit.py") + assert entry.__name__ == "audit_for_knowledge" + result = run_release_audit( + adapter, "release_audit.py", + upstream_repo=fixture["upstream"], from_ref=str(fixture["old"]), + to_ref=str(fixture["new"]), knowledge_root=fixture["knowledge"], + project_root=fixture["project"], + ) + assert result.to_sha == fixture["new"] + kinds = {i["kind"] for i in result.issues} + # the real adapter baseline is used, so its owner documents are absent + # from this fixture project; only knowledge-document kinds are enforced + assert "stale_knowledge_source" in kinds + assert kinds <= {"stale_knowledge_source", "knowledge_metadata_error", "owner_document_missing"} + for bad in ("../x.py", "/abs.py", "release_baseline.yaml", "missing.py"): + import pytest + with pytest.raises(ReleaseAuditPluginError): + load_release_auditor(adapter, bad) + + +def test_release_audit_plugin_ships_in_the_adapter_tree(): + root = Path(__file__).resolve().parents[1] + assert (root / "adapters" / "vllm_omni" / "release_audit.py").is_file() + import tools.vllm_omni_release_audit as shim + assert shim.audit_release.__module__ == "infermatrix_adapter_vllm_omni_release_audit" + + +def test_generated_mode_reports_a_differing_previous_sha(tmp_path): + fixture = _fixture(tmp_path) + data = yaml.safe_load(fixture["baseline"].read_text(encoding="utf-8")) + data["upstream"]["previous_audited_sha"] = "f" * 40 + fixture["baseline"].write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + assert "baseline_from_mismatch" in {i["kind"] for i in _audit(fixture).data["issues"]} + generated = _generated(fixture) + assert {"kind": "committed_previous_differs", "baseline": "f" * 40, + "target": fixture["old"]} in generated.data["reconciliation"] + + +def test_plugin_loader_rejects_malformed_reports(): + import pytest + from infermatrix_copilot.knowledge_service.release_audit import ( + ReleaseAuditPluginError, _result, + ) + + good = { + "upstream": {"from": {"sha": "a" * 40}, "to": {"sha": "b" * 40}}, + "issues": [], "reconciliation": [], + "generated_baseline": {"upstream": {"audited_sha": "b" * 40}}, + } + assert _result(good).enforced_clean + broken = [ + {**good, "issues": ""}, + {**good, "reconciliation": {}}, + {**good, "issues": [{"no_kind": 1}]}, + {**good, "upstream": {"from": {"sha": None}, "to": {"sha": "b" * 40}}}, + {k: v for k, v in good.items() if k != "generated_baseline"}, + {**good, "generated_baseline": {"upstream": {"audited_sha": "c" * 40}}}, + "not a mapping", + ] + for report in broken: + with pytest.raises(ReleaseAuditPluginError): + _result(report) diff --git a/tools/vllm_omni_release_audit.py b/tools/vllm_omni_release_audit.py index 9c9583d84..3a392c7e8 100644 --- a/tools/vllm_omni_release_audit.py +++ b/tools/vllm_omni_release_audit.py @@ -1,751 +1,27 @@ -"""Deterministic, read-only drift audit for vLLM-Omni releases.""" +"""Compatibility entry for the vLLM-Omni release audit. + +The implementation is the adapter plugin ``adapters/vllm_omni/release_audit.py`` +(packaged with the wheel so the knowledge service can load it). This module +re-exports it for the CLI wrapper, CI and tests. +""" from __future__ import annotations -import argparse -import ast -import fnmatch -import hashlib -import json -import re -import subprocess -from dataclasses import dataclass +import importlib.util +import sys from pathlib import Path -from typing import Any, Iterable - -import yaml - - -_REPO_ROOT = Path(__file__).resolve().parents[1] -_REGISTRIES = { - "autoregressive": ( - "vllm_omni/model_executor/models/registry.py", - "_OMNI_MODELS", - True, - ), - "diffusion": ( - "vllm_omni/diffusion/registry.py", - "_DIFFUSION_MODELS", - True, - ), - "pipelines": ( - "vllm_omni/config/pipeline_registry.py", - "OMNI_PIPELINES", - False, - ), -} -_UPSTREAM_SOURCE_PREFIXES = ( - ".buildkite/", - "benchmarks/", - "docs/", - "tests/", - "vllm_omni/", -) -_SOURCE_EXCLUDED_PARTS = {"_archive", "history", "incidents", "results"} -_PIN_RE = re.compile( - r"\b(?:main|v\d[0-9A-Za-z.-]*)\s*@\s*`?([0-9a-f]{8,40})", - re.IGNORECASE, -) - - -class ReleaseAuditError(RuntimeError): - """The audit could not read or understand one of its declared inputs.""" - - -@dataclass(frozen=True) -class ChangedPath: - status: str - path: str - old_path: str | None = None - - def as_dict(self) -> dict[str, str]: - item = {"status": self.status, "path": self.path} - if self.old_path: - item["old_path"] = self.old_path - return item - - -@dataclass(frozen=True) -class ReleaseAuditReport: - data: dict[str, Any] - - @property - def has_drift(self) -> bool: - return bool(self.data["issues"]) - - def to_json(self) -> str: - return ( - json.dumps( - self.data, - ensure_ascii=False, - indent=2, - sort_keys=True, - ) - + "\n" - ) - - -class GitSnapshotReader: - """Read immutable Git objects without checking out or importing upstream.""" - - def __init__(self, repo: Path | str): - self.repo = Path(repo).resolve() - if not self.repo.is_dir(): - raise ReleaseAuditError(f"upstream checkout does not exist: {self.repo}") - self._run_text("rev-parse", "--git-dir") - - def _run( - self, - *args: str, - text: bool, - check: bool = True, - ) -> subprocess.CompletedProcess: - try: - result = subprocess.run( - ["git", "-C", str(self.repo), *args], - capture_output=True, - text=text, - encoding="utf-8" if text else None, - errors="replace" if text else None, - timeout=60, - check=False, - ) - except (OSError, subprocess.SubprocessError) as exc: - raise ReleaseAuditError(f"git {' '.join(args)} failed: {exc}") from exc - if check and result.returncode: - stderr = ( - result.stderr.strip() - if text - else result.stderr.decode("utf-8", errors="replace").strip() - ) - raise ReleaseAuditError( - f"git {' '.join(args)} failed ({result.returncode}): {stderr}" - ) - return result - - def _run_text(self, *args: str, check: bool = True) -> str: - return self._run(*args, text=True, check=check).stdout.strip() - - def resolve(self, ref: str) -> str: - sha = self._run_text("rev-parse", "--verify", f"{ref}^{{commit}}") - if not re.fullmatch(r"[0-9a-f]{40}", sha): - raise ReleaseAuditError(f"ref did not resolve to a commit: {ref}") - return sha - - def read_text(self, commit: str, path: str) -> str: - return self._run_text("show", f"{commit}:{path}") - - def tree_files(self, commit: str) -> list[str]: - raw = self._run( - "ls-tree", - "-r", - "--name-only", - "-z", - commit, - text=False, - ).stdout - return sorted( - part.decode("utf-8", errors="surrogateescape") - for part in raw.split(b"\0") - if part - ) - - def changed_paths(self, old: str, new: str) -> list[ChangedPath]: - raw = self._run( - "diff", - "--name-status", - "-z", - "-M", - old, - new, - "--", - text=False, - ).stdout - fields = [ - part.decode("utf-8", errors="surrogateescape") - for part in raw.split(b"\0") - if part - ] - changed: list[ChangedPath] = [] - index = 0 - while index < len(fields): - status = fields[index] - index += 1 - kind = status[:1] - if kind in {"R", "C"}: - if index + 1 >= len(fields): - raise ReleaseAuditError("unexpected truncated git rename output") - old_path, new_path = fields[index], fields[index + 1] - index += 2 - changed.append( - ChangedPath(status=status, path=new_path, old_path=old_path) - ) - else: - if index >= len(fields): - raise ReleaseAuditError("unexpected truncated git diff output") - changed.append(ChangedPath(status=status, path=fields[index])) - index += 1 - return sorted(changed, key=lambda item: (item.path, item.old_path or "")) - - -def _assigned_dict(source: str, variable: str, path: str) -> ast.Dict: - try: - tree = ast.parse(source, filename=path) - except SyntaxError as exc: - raise ReleaseAuditError(f"cannot parse {path}: {exc}") from exc - for node in tree.body: - value: ast.AST | None = None - if isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == variable - for target in node.targets - ): - value = node.value - elif ( - isinstance(node, ast.AnnAssign) - and isinstance(node.target, ast.Name) - and node.target.id == variable - ): - value = node.value - if value is not None: - if not isinstance(value, ast.Dict): - raise ReleaseAuditError(f"{path}:{variable} is not a dict literal") - return value - raise ReleaseAuditError(f"{path} does not define {variable}") - - -def _registry_entries( - source: str, - variable: str, - path: str, - include_targets: bool, -) -> dict[str, list[str]] | list[str]: - dictionary = _assigned_dict(source, variable, path) - entries: dict[str, list[str]] = {} - keys: list[str] = [] - for key_node, value_node in zip(dictionary.keys, dictionary.values): - if not isinstance(key_node, ast.Constant) or not isinstance( - key_node.value, str - ): - raise ReleaseAuditError(f"{path}:{variable} has a non-string key") - key = key_node.value - if key in keys: - raise ReleaseAuditError(f"{path}:{variable} has duplicate key {key!r}") - keys.append(key) - if include_targets: - try: - value = ast.literal_eval(value_node) - except (ValueError, TypeError, SyntaxError) as exc: - raise ReleaseAuditError( - f"{path}:{variable}[{key!r}] is not a literal target tuple" - ) from exc - if ( - not isinstance(value, tuple) - or len(value) != 3 - or not all(isinstance(part, str) for part in value) - ): - raise ReleaseAuditError( - f"{path}:{variable}[{key!r}] is not a 3-string target tuple" - ) - entries[key] = list(value) - return dict(sorted(entries.items())) if include_targets else sorted(keys) - - -def _fingerprint(value: Any) -> dict[str, int | str]: - encoded = json.dumps( - value, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ).encode("utf-8") - return { - "count": len(value), - "sha256": hashlib.sha256(encoded).hexdigest(), - } - - -def snapshot_inventory(reader: GitSnapshotReader, commit: str) -> dict[str, Any]: - inventories: dict[str, Any] = {} - for name, (path, variable, include_targets) in _REGISTRIES.items(): - inventories[name] = _registry_entries( - reader.read_text(commit, path), - variable, - path, - include_targets, - ) - inventories["deploy_yamls"] = sorted( - path - for path in reader.tree_files(commit) - if path.startswith("vllm_omni/deploy/") - and path.casefold().endswith((".yaml", ".yml")) - ) - return inventories - - -def _inventory_delta(old: Any, new: Any) -> dict[str, Any]: - if isinstance(old, dict) and isinstance(new, dict): - old_keys, new_keys = set(old), set(new) - return { - "added": sorted(new_keys - old_keys), - "removed": sorted(old_keys - new_keys), - "changed": sorted( - key for key in old_keys & new_keys if old[key] != new[key] - ), - } - old_set, new_set = set(old), set(new) - return { - "added": sorted(new_set - old_set), - "removed": sorted(old_set - new_set), - "changed": [], - } - - -def _load_yaml_mapping(path: Path, label: str) -> dict[str, Any]: - try: - value = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - except (OSError, yaml.YAMLError) as exc: - raise ReleaseAuditError(f"cannot read {label} {path}: {exc}") from exc - if not isinstance(value, dict): - raise ReleaseAuditError(f"{label} must be a YAML mapping: {path}") - return value - - -def _matches(path: str, pattern: str) -> bool: - normalized = pattern.replace("\\", "/") - if normalized.endswith("/"): - return path.startswith(normalized) - return fnmatch.fnmatchcase(path, normalized) - - -def _route_path(path: str, baseline: dict[str, Any]) -> tuple[list[str], list[str]]: - owners = sorted( - owner - for owner, patterns in (baseline.get("path_owners") or {}).items() - if any(_matches(path, str(pattern)) for pattern in patterns or []) - ) - ignored = sorted( - str(item.get("reason") or "no reason supplied") - for item in baseline.get("ignored_paths") or [] - if isinstance(item, dict) and _matches(path, str(item.get("pattern") or "")) - ) - return owners, ignored - - -def _adapter_modules(path: str, manifest: dict[str, Any]) -> list[str]: - modules: list[str] = [] - for module, spec in (manifest.get("modules") or {}).items(): - for pattern in (spec or {}).get("local_paths") or []: - prefix = str(pattern).replace("\\", "/").rstrip("*").rstrip("/") - if path.startswith(prefix): - modules.append(str(module)) - break - return sorted(modules) - - -def _source_path(value: str) -> str | None: - candidate = value.strip().strip("`") - if not candidate.startswith(_UPSTREAM_SOURCE_PREFIXES): - return None - candidate = candidate.split("::", 1)[0] - candidate = re.sub(r":\d+(?:-\d+)?$", "", candidate) - return candidate - - -def _path_exists(files: Iterable[str], source_path: str) -> bool: - file_list = files if isinstance(files, list) else list(files) - if any(char in source_path for char in "*?["): - return any(fnmatch.fnmatchcase(path, source_path) for path in file_list) - if source_path.endswith("/"): - return any(path.startswith(source_path) for path in file_list) - return source_path in file_list or any( - path.startswith(source_path.rstrip("/") + "/") for path in file_list - ) - - -def _knowledge_source_issues( - knowledge_root: Path, - old_files: list[str], - new_files: list[str], - changes: list[ChangedPath], -) -> list[dict[str, str]]: - repo_root = knowledge_root / "repos" / "vllm-omni" - if not repo_root.is_dir(): - raise ReleaseAuditError(f"vLLM-Omni knowledge root is missing: {repo_root}") - rename_map = { - item.old_path: item.path - for item in changes - if item.old_path and item.status.startswith("R") - } - issues: list[dict[str, str]] = [] - for page in sorted(repo_root.rglob("*.md")): - if any( - part in _SOURCE_EXCLUDED_PARTS for part in page.relative_to(repo_root).parts - ): - continue - text = page.read_text(encoding="utf-8") - if not text.startswith("---"): - continue - parts = text.split("---", 2) - if len(parts) != 3: - continue - try: - metadata = yaml.safe_load(parts[1]) or {} - except yaml.YAMLError as exc: - issues.append( - { - "kind": "knowledge_metadata_error", - "document": page.relative_to(knowledge_root.parent).as_posix(), - "detail": str(exc).splitlines()[0], - } - ) - continue - sources = metadata.get("sources") or [] - if not isinstance(sources, list): - issues.append( - { - "kind": "knowledge_metadata_error", - "document": page.relative_to(knowledge_root.parent).as_posix(), - "detail": "sources must be a list", - } - ) - continue - for raw_source in sources: - if not isinstance(raw_source, str): - continue - source_path = _source_path(raw_source) - if not source_path: - continue - if _path_exists(old_files, source_path) and not _path_exists( - new_files, source_path - ): - issue = { - "kind": "stale_knowledge_source", - "document": page.relative_to(knowledge_root.parent).as_posix(), - "source": source_path, - } - if source_path in rename_map: - issue["renamed_to"] = rename_map[source_path] - issues.append(issue) - return sorted( - issues, - key=lambda item: ( - item["kind"], - item.get("document", ""), - item.get("source", ""), - ), - ) - - -def _pin_issues( - project_root: Path, - baseline: dict[str, Any], - audited_sha: str, -) -> list[dict[str, str]]: - issues: list[dict[str, str]] = [] - for rel in baseline.get("pin_documents") or []: - document = project_root / str(rel) - if not document.is_file(): - issues.append( - { - "kind": "pin_document_missing", - "document": str(rel), - } - ) - continue - pins = sorted(set(_PIN_RE.findall(document.read_text(encoding="utf-8")))) - if not pins: - issues.append( - { - "kind": "pin_marker_missing", - "document": str(rel), - } - ) - continue - for pin in pins: - if not audited_sha.startswith(pin): - issues.append( - { - "kind": "stale_pin", - "document": str(rel), - "actual": pin, - "expected": audited_sha, - } - ) - return issues - - -def _policy_issues( - project_root: Path, - baseline: dict[str, Any], - target_files: list[str], -) -> list[dict[str, str]]: - issues: list[dict[str, str]] = [] - owner_documents = baseline.get("owner_documents") or {} - for owner, patterns in sorted((baseline.get("path_owners") or {}).items()): - documents = owner_documents.get(owner) or [] - if not documents: - issues.append({"kind": "owner_document_missing", "owner": str(owner)}) - for document in documents: - if not (project_root / str(document)).is_file(): - issues.append( - { - "kind": "owner_document_missing", - "owner": str(owner), - "document": str(document), - } - ) - for pattern in patterns or []: - if not _path_exists(target_files, str(pattern)): - issues.append( - { - "kind": "stale_source_map_path", - "owner": str(owner), - "path": str(pattern), - } - ) - return issues - - -def audit_release( - *, - upstream_repo: Path | str, - from_ref: str, - to_ref: str, - baseline_path: Path | str, - adapter_manifest_path: Path | str, - knowledge_root: Path | str, - project_root: Path | str = _REPO_ROOT, -) -> ReleaseAuditReport: - reader = GitSnapshotReader(upstream_repo) - old_sha, new_sha = reader.resolve(from_ref), reader.resolve(to_ref) - baseline_path = Path(baseline_path) - baseline = _load_yaml_mapping(baseline_path, "release baseline") - manifest = _load_yaml_mapping(Path(adapter_manifest_path), "adapter manifest") - if baseline.get("schema_version") != 1: - raise ReleaseAuditError("release baseline schema_version must be 1") - - old_inventory = snapshot_inventory(reader, old_sha) - new_inventory = snapshot_inventory(reader, new_sha) - changes = reader.changed_paths(old_sha, new_sha) - old_files, new_files = reader.tree_files(old_sha), reader.tree_files(new_sha) - - issues: list[dict[str, Any]] = [] - expected_sha = str((baseline.get("upstream") or {}).get("audited_sha") or "") - if expected_sha != new_sha: - issues.append( - { - "kind": "baseline_pin_mismatch", - "baseline": expected_sha, - "target": new_sha, - } - ) - expected_old_sha = str( - (baseline.get("upstream") or {}).get("previous_audited_sha") or "" - ) - if expected_sha == new_sha and expected_old_sha != old_sha: - issues.append( - { - "kind": "baseline_from_mismatch", - "baseline": expected_old_sha, - "target": old_sha, - } - ) - - actual_fingerprints = { - name: _fingerprint(value) for name, value in new_inventory.items() - } - expected_fingerprints = baseline.get("inventories") or {} - for name in sorted(actual_fingerprints): - if expected_fingerprints.get(name) != actual_fingerprints[name]: - issues.append( - { - "kind": "inventory_mismatch", - "inventory": name, - "baseline": expected_fingerprints.get(name), - "target": actual_fingerprints[name], - } - ) - - routing: list[dict[str, Any]] = [] - adapter_uncovered: list[str] = [] - for change in changes: - owners, ignored = _route_path(change.path, baseline) - adapter_modules = _adapter_modules(change.path, manifest) - route = { - **change.as_dict(), - "owners": owners, - "ignored": ignored, - "adapter_modules": adapter_modules, - "knowledge_documents": sorted( - { - str(document) - for owner in owners - for document in (baseline.get("owner_documents") or {}).get( - owner, [] - ) - } - ), - } - routing.append(route) - if change.path.startswith("vllm_omni/") and not adapter_modules: - adapter_uncovered.append(change.path) - if not owners and not ignored: - issues.append({"kind": "unmatched_path", "path": change.path}) - elif len(owners) > 1 or (owners and ignored): - issues.append( - { - "kind": "suspicious_path_route", - "path": change.path, - "owners": owners, - "ignored": ignored, - } - ) - - issues.extend( - _knowledge_source_issues( - Path(knowledge_root), - old_files, - new_files, - changes, - ) - ) - issues.extend(_pin_issues(Path(project_root), baseline, new_sha)) - issues.extend( - _policy_issues( - Path(project_root), - baseline, - new_files, - ) - ) - issues = sorted( - issues, - key=lambda item: ( - str(item.get("kind", "")), - str(item.get("path", "")), - str(item.get("document", "")), - str(item.get("inventory", "")), - ), - ) - - deltas = { - name: _inventory_delta(old_inventory[name], new_inventory[name]) - for name in sorted(new_inventory) - } - status_counts: dict[str, int] = {} - for change in changes: - kind = change.status[:1] - status_counts[kind] = status_counts.get(kind, 0) + 1 - - data = { - "schema_version": 1, - "upstream": { - "repository": str((baseline.get("upstream") or {}).get("repository") or ""), - "from": {"ref": from_ref, "sha": old_sha}, - "to": {"ref": to_ref, "sha": new_sha}, - }, - "inventory": { - "deltas": deltas, - "target_fingerprints": actual_fingerprints, - }, - "paths": { - "counts": dict(sorted(status_counts.items())), - "changes": [item.as_dict() for item in changes], - "routing": routing, - "adapter_uncovered": sorted(set(adapter_uncovered)), - }, - "issues": issues, - "result": "drift" if issues else "clean", - } - return ReleaseAuditReport(data) - - -def render_summary(report: ReleaseAuditReport) -> str: - data = report.data - old = data["upstream"]["from"]["sha"][:8] - new = data["upstream"]["to"]["sha"][:8] - deltas = data["inventory"]["deltas"] - lines = [f"vLLM-Omni release audit: {old} -> {new}"] - for name in ("autoregressive", "diffusion", "pipelines", "deploy_yamls"): - delta = deltas[name] - lines.append( - f" {name}: +{len(delta['added'])} -{len(delta['removed'])} " - f"~{len(delta['changed'])}" - ) - lines.append( - f" changed paths: {len(data['paths']['changes'])}; " - "runtime-manifest gaps (reported separately): " - f"{len(data['paths']['adapter_uncovered'])}" - ) - lines.append(f" issues: {len(data['issues'])}") - for issue in data["issues"][:10]: - subject = ( - issue.get("path") or issue.get("document") or issue.get("inventory") or "" - ) - lines.append(f" - {issue['kind']}: {subject}".rstrip()) - if len(data["issues"]) > 10: - lines.append(f" ... {len(data['issues']) - 10} more in JSON report") - lines.append(f"RESULT: {data['result'].upper()}") - return "\n".join(lines) - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Audit vLLM-Omni release drift without modifying either repo." - ) - parser.add_argument("--from", dest="from_ref", required=True) - parser.add_argument("--to", dest="to_ref", required=True) - parser.add_argument("--repo", required=True, type=Path) - parser.add_argument( - "--baseline", - type=Path, - default=_REPO_ROOT / "adapters" / "vllm_omni" / "release_baseline.yaml", - ) - parser.add_argument( - "--adapter-manifest", - type=Path, - default=_REPO_ROOT / "adapters" / "vllm_omni" / "manifest.yaml", - ) - parser.add_argument( - "--knowledge-root", - type=Path, - default=_REPO_ROOT / "knowledge", - ) - parser.add_argument( - "--project-root", - type=Path, - default=_REPO_ROOT, - help="root used to resolve pin_documents from the baseline", - ) - parser.add_argument("--json-output", type=Path) - parser.add_argument( - "--mode", - choices=("enforce", "report-only"), - default="enforce", - ) - return parser - - -def main(argv: list[str] | None = None) -> int: - args = build_parser().parse_args(argv) - try: - report = audit_release( - upstream_repo=args.repo, - from_ref=args.from_ref, - to_ref=args.to_ref, - baseline_path=args.baseline, - adapter_manifest_path=args.adapter_manifest, - knowledge_root=args.knowledge_root, - project_root=args.project_root, - ) - except ReleaseAuditError as exc: - print(f"release audit error: {exc}") - return 2 - print(render_summary(report)) - if args.json_output: - args.json_output.parent.mkdir(parents=True, exist_ok=True) - args.json_output.write_text(report.to_json(), encoding="utf-8") - print(f"JSON report: {args.json_output}") - return 1 if report.has_drift and args.mode == "enforce" else 0 +_PLUGIN = Path(__file__).resolve().parents[1] / "adapters" / "vllm_omni" / "release_audit.py" +_NAME = "infermatrix_adapter_vllm_omni_release_audit" +if _NAME not in sys.modules: + _spec = importlib.util.spec_from_file_location(_NAME, _PLUGIN) + _module = importlib.util.module_from_spec(_spec) + sys.modules[_NAME] = _module + _spec.loader.exec_module(_module) +_module = sys.modules[_NAME] +globals().update({ + name: getattr(_module, name) for name in dir(_module) if not name.startswith("__") +}) if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(main()) # noqa: F821 - re-exported above