From b2c868c6ecf115e15edbc5332a56e622311d926b Mon Sep 17 00:00:00 2001 From: tzhouam Date: Tue, 6 Oct 2026 14:58:46 +0800 Subject: [PATCH 1/2] Add portable knowledge onboarding and autonomous feature discovery --- doc/architecture/SPEC/app/run_service.md | 9 +- doc/architecture/SPEC/config.md | 7 +- doc/architecture/SPEC/direct_routing.md | 14 +- doc/architecture/SPEC/engine/agent_runtime.md | 9 +- doc/architecture/SPEC/kb_service.md | 10 + doc/architecture/SPEC/knowledge_context.md | 45 ++ doc/architecture/SPEC/knowledge_docs.md | 14 +- doc/architecture/SPEC/sdk.md | 18 +- doc/architecture/SPEC/thin_mcp_server.md | 16 +- doc/architecture/feature-discovery.md | 62 +++ doc/architecture/portable-kb.md | 94 ++++ .../templates/kb-init-new-repository.yaml | 2 + eval/knowledge-depth/audit_depth_retrieval.py | 13 + knowledge/tools/check_knowledge_tree.py | 2 +- knowledge/tools/check_wiki_lint.py | 2 +- src/infermatrix_copilot/app/run_service.py | 18 +- src/infermatrix_copilot/config.py | 2 + src/infermatrix_copilot/direct_routing.py | 21 +- .../engine/agent_runtime/knowledge.py | 74 ++- .../engine/agent_runtime/runner.py | 18 +- src/infermatrix_copilot/kb_service/cli.py | 44 +- .../kb_service/depth_index.py | 2 +- .../kb_service/depth_inputs.py | 15 +- .../kb_service/evidence_bundle.py | 92 ++++ .../kb_service/feature_discovery_index.py | 142 +++++- .../kb_service/git_source.py | 277 +++++++++++ .../kb_service/init_feature_discovery.py | 389 +++++++++++++--- .../kb_service/init_knowledge.py | 38 +- .../kb_service/init_knowledge_depth.py | 23 +- .../kb_service/init_stages.py | 30 +- .../kb_service/init_support.py | 41 +- .../kb_service/knowledge_coverage.py | 15 +- .../kb_service/knowledge_depth.py | 5 +- .../kb_service/knowledge_store.py | 329 +++++++++++++ .../kb_service/model_dispatch.py | 89 ++++ src/infermatrix_copilot/kb_service/models.py | 17 + .../kb_service/native_depth_audit.py | 431 +++++++++++++++++ .../kb_service/portable_commands.py | 429 +++++++++++++++++ .../kb_service/portable_init.py | 211 +++++++++ .../kb_service/portable_publication.py | 50 ++ .../kb_service/repo_spec.py | 427 +++++++++++++++++ .../kb_service/source_links.py | 17 + src/infermatrix_copilot/knowledge_context.py | 339 ++++++++++++++ src/infermatrix_copilot/knowledge_docs.py | 52 ++- .../knowledge_service/facts.py | 2 +- .../knowledge_service/lifecycle.py | 2 +- .../knowledge_service/pinned_claims.py | 2 +- src/infermatrix_copilot/sdk/v1/__init__.py | 2 + src/infermatrix_copilot/sdk/v1/direct.py | 112 ++++- src/infermatrix_copilot/sdk/v1/models.py | 12 +- src/infermatrix_copilot/thin_mcp_server.py | 47 ++ test/test_evidence_bundle.py | 39 ++ test/test_kb_depth_retrieval_audit.py | 23 + test/test_kb_discovery_contract_units.py | 234 ++++++++++ test/test_kb_feature_discovery.py | 10 +- test/test_kb_feature_discovery_stage.py | 7 + test/test_kb_model_dispatch.py | 157 +++++++ test/test_kb_portable_knowledge_store.py | 262 +++++++++++ test/test_kb_portable_repositories.py | 258 +++++++++++ test/test_knowledge_context.py | 204 ++++++++ test/test_knowledge_retrieval.py | 46 +- test/test_portable_commands.py | 85 ++++ test/test_portable_init_flow.py | 400 ++++++++++++++++ test/test_portable_publication.py | 89 ++++ test/test_thin_mcp_server.py | 5 + tools/audit_native_depth_approvals.py | 435 +----------------- 66 files changed, 5773 insertions(+), 614 deletions(-) create mode 100644 doc/architecture/SPEC/knowledge_context.md create mode 100644 doc/architecture/feature-discovery.md create mode 100644 doc/architecture/portable-kb.md create mode 100644 src/infermatrix_copilot/kb_service/evidence_bundle.py create mode 100644 src/infermatrix_copilot/kb_service/git_source.py create mode 100644 src/infermatrix_copilot/kb_service/knowledge_store.py create mode 100644 src/infermatrix_copilot/kb_service/model_dispatch.py create mode 100644 src/infermatrix_copilot/kb_service/native_depth_audit.py create mode 100644 src/infermatrix_copilot/kb_service/portable_commands.py create mode 100644 src/infermatrix_copilot/kb_service/portable_init.py create mode 100644 src/infermatrix_copilot/kb_service/portable_publication.py create mode 100644 src/infermatrix_copilot/kb_service/repo_spec.py create mode 100644 src/infermatrix_copilot/kb_service/source_links.py create mode 100644 src/infermatrix_copilot/knowledge_context.py create mode 100644 test/test_evidence_bundle.py create mode 100644 test/test_kb_discovery_contract_units.py create mode 100644 test/test_kb_model_dispatch.py create mode 100644 test/test_kb_portable_knowledge_store.py create mode 100644 test/test_kb_portable_repositories.py create mode 100644 test/test_knowledge_context.py create mode 100644 test/test_portable_commands.py create mode 100644 test/test_portable_init_flow.py create mode 100644 test/test_portable_publication.py diff --git a/doc/architecture/SPEC/app/run_service.md b/doc/architecture/SPEC/app/run_service.md index ff3d21adb..a062a4897 100644 --- a/doc/architecture/SPEC/app/run_service.md +++ b/doc/architecture/SPEC/app/run_service.md @@ -1,6 +1,6 @@ # app/run_service.py —— 规范 - + `RunService` is the durable application boundary shared by the embedded Strict SDK and MCP transport. It owns policy-checked reserve/start, a queue @@ -8,6 +8,13 @@ drained by `STRICT_MAX_WORKERS` workers (default 1), isolated child launch, star readiness, bounded status/result polling, and repository-scoped knowledge reads. Construction and polling require no CLI or MCP module import. +Knowledge reads resolve repository slices from the same frozen view's registry, +with legacy adapters retained as fallback. Search/read always pass the view's +manifest verifier. Child launch forwards the host-only +`ALLOWED_KNOWLEDGE_REPOSITORIES`; tool input cannot grant cross-repository access. +Strict adaptive document tools share the run's `KnowledgeContextService` ledger; +legacy standalone document endpoints retain their per-call read windows. + Reservation and run ID containment go directly through `RunReservation`, not the workflow `Copilot` facade. `Copilot` remains for playbook readiness and workflow execution; it does not mediate the service's reserve or poll diff --git a/doc/architecture/SPEC/config.md b/doc/architecture/SPEC/config.md index e381657f6..75775a70c 100644 --- a/doc/architecture/SPEC/config.md +++ b/doc/architecture/SPEC/config.md @@ -1,6 +1,6 @@ # config.py —— 规范 - + `LOC ~658 · 配置 · refactor-status: oversized` @@ -26,6 +26,11 @@ worker 数 `strict_max_workers`（`STRICT_MAX_WORKERS`，默认 1，启动时校 以带标记的 JSON 评论贴到该锁定的 mailbox issue，供另一台主机上的 reviewbot 摄取）。 ## 公开契约 + +`allowed_knowledge_repositories`（`ALLOWED_KNOWLEDGE_REPOSITORIES`，默认空 tuple）是 +运行主机授权读取的跨仓库 ID。它不进入模型工具的权限参数；授权仍须与固定快照中的 +confirmed 依赖及源码/目录 pin 一同验证，空配置只允许当前仓库。 + 带全部可调项的 `Settings`；`reviewer` / `intent`（回退到 `agent_model`）； `repo_path(name)`；`model_for(mode)`；`tier_target(role)` → `ResolvedTarget`； `expansion_env()`；`knowledge_runtime_repos`；`allowed_repo_roots` diff --git a/doc/architecture/SPEC/direct_routing.md b/doc/architecture/SPEC/direct_routing.md index 846ebe34d..035c810e3 100644 --- a/doc/architecture/SPEC/direct_routing.md +++ b/doc/architecture/SPEC/direct_routing.md @@ -1,13 +1,13 @@ # direct_routing.py —— 规范 - + `LOC ~880 · Direct 模式完整策略包与仓库中立的知识路由 · refactor-status: stable` ## 职责 Direct 模式的知识路由**机制**。模块中不出现任何被服务仓库的名字：每个仓库的 -owner/model 路由表是知识数据 `knowledge/repos//_routes.yaml`，仓库别名 -来自 `adapters//manifest.yaml` 的 `repo.full_name`/`repo.aliases`。 +owner/model 路由表是知识数据 `knowledge/repos//_routes.yaml`，仓库身份和别名 +优先来自同一固定视图的 `_repositories.yaml`，旧 adapter 别名保持兼容。 公开面由 `contract.py` 再导出。 ## 公开契约 @@ -39,7 +39,9 @@ Direct policy bundle）、`direct_knowledge_routes`、 为模块级 `__getattr__` 惰性别名。 - **路由选择按数据而非仓库名**：仓库有非空 `_routes.yaml` → title/body owner 路由 + 模型路由 + scope fallback；否则有 adapter → changed-file 路由；否则 - `unsupported_exact_router`。`_routes.yaml` 引用的页面不存在时 fail-closed。 + `unsupported_exact_router`。已注册但未生成 owner routes 的仓库返回显式 + `description_unrouted`，仍可检索自己的说明知识；不得借用其他仓库路由。 + `_routes.yaml` 引用的页面不存在时 fail-closed。 - `repo` 必填：空值抛 `ValueError`，不再默认某个仓库；形似路径的仓库名直接 视为不支持。 - **changed files 校验选择、绝不静默替换选择**：title/body 选 owner， @@ -71,7 +73,9 @@ stdlib + PyYAML + `.adapters`（AdapterError / AdapterRegistry / RepoAdapter）+ ## 扩展点 新 owner 路由/模型规则 → 改对应仓库的 `_routes.yaml`（schema_version 1： `owners[{owner, path, signals, scope_prefixes}]`、可选 `models{dir, page}`）； -新仓库 → 提供 adapter 与 `_routes.yaml`，不改 `src/`。 +新仓库 → 提供固定视图 registry 与 `_routes.yaml`，不要求打包 adapter、不改 `src/`。 +已有 adapter 仓库继续兼容。自适应 SDK 通过统一会话对本模块选出的规则/导航再计量； +旧 bundle 的 6,000 字符相关背景预算保持不变。 ## 测试 `test_knowledge_view_routing.py`（150 个真实 vllm-omni PR 的黄金路由输出、 diff --git a/doc/architecture/SPEC/engine/agent_runtime.md b/doc/architecture/SPEC/engine/agent_runtime.md index 8e75fe2ae..2ba512131 100644 --- a/doc/architecture/SPEC/engine/agent_runtime.md +++ b/doc/architecture/SPEC/engine/agent_runtime.md @@ -1,11 +1,18 @@ # engine/agent_runtime/ —— 规范 - + `LOC ~1690（7 个文件） · 引擎（受治理的 agent 运行时） · refactor-status: ok` ## 职责 +含 `_repositories.yaml` 的固定快照默认采用 `knowledge_context_profile="adaptive"`； +旧快照默认 legacy，也可显式选择。`doc_search` / `doc_read` / `doc_related`、规则与 +`doc_context_briefing` 共用 run 私有累计会话，`doc_context_expand` 必须给原因且保留 +源码/输出留白。实际注入正文和哈希写 trace，runner直接注入已围栏的 `model_content`。 +跨仓库 target 只选择已确认依赖，授权由 Settings 固定传入；模型工具不能授予权限。 + + 开启 `profile_briefing_enabled` 且证据包含 `pr_diff` 时，runner 使用既有 diff parser 提取 路径，通过 `doc_related` 在首个模型调用前注入同一有界知识背景。内容放入 untrusted 围栏并转义标签；`review_knowledge_context` 记录命中页面、状态和正文长度。 diff --git a/doc/architecture/SPEC/kb_service.md b/doc/architecture/SPEC/kb_service.md index 5ef366abe..39319024c 100644 --- a/doc/architecture/SPEC/kb_service.md +++ b/doc/architecture/SPEC/kb_service.md @@ -5,6 +5,16 @@ `知识服务核心：仓库配置、账本、outbox、CLI · refactor-status: new` ## 职责 +- 通用接入：`git_source` 固定任意 Git 来源，`repo_spec` 保存经明确接受的范围； + `portable_init` 创建本地受审工作区，`portable_commands` 提供 onboard、register、 + 目录接受、发布和增量批次准备，不伪造已合并 PR。见 [通用知识库流程](../portable-kb.md)。 +- 功能发现：文档轮、源码轮、独立评审、最多两轮残留补查；共享完整库存和入口/契约单元， + 单项最多三次修正，失败保持未知。`evidence_bundle` 将固定哈希行段传给后续阶段。 +- 存储与接受：`knowledge_store` 单一主存储加只读镜像；`portable_publication` 区分 foundation + 与 final，默认只激活真实最终回执，显式 partial 仍为未完成。最终复核完整生产库存、 + 七维分别 >90%、每功能认可和 ≥85% 结构覆盖，原生认可由 `native_depth_audit` 重放。 +- `source_links` 为 GitHub、GitLab 和本地 `repo://` 源码生成固定版本引用；原 GitHub 引用 + 与历史认可正文保持一致。通用知识 pin 支持 SHA‑1/SHA‑256。 - `config`：解析 adapter `knowledge_lifecycle`（未知键、非法模式/触发器、缺失审计插件、 知识目录与 adapter 不一致、私有上游配 `auto_merge`、`auto_merge` 无校准集均抛 `LifecycleConfigError`）；`load_registry` 返回按仓库名的注册表，`general` 由服务配置。 diff --git a/doc/architecture/SPEC/knowledge_context.md b/doc/architecture/SPEC/knowledge_context.md new file mode 100644 index 000000000..6718ca050 --- /dev/null +++ b/doc/architecture/SPEC/knowledge_context.md @@ -0,0 +1,45 @@ +# knowledge_context.py —— 规范 + + + +## 职责与公开契约 + +`KnowledgeContextService` 是 SDK、MCP 与 Strict 共用的、固定快照的知识上下文服务。 +`ContextBudget` 声明模型窗口与留白；`ContextError` 拒绝身份、权限或预算不一致。 +`open_session` / `status` / `expand` 管理会话；`related` / `search` / `read` / `inject` +返回同一累计预算下的实际正文和引用。服务不调用模型、不改知识页、不宣称评审准确率。 + +## 身份与权限 + +会话身份绑定快照树哈希、仓库 ID、知识切片、源码 pin、目录和政策哈希、PR 源码 pin、 +review ID、计量器及预算配置。已验证快照每次读取复核 manifest；开发目录绑定真实文件 +内容摘要。恢复复用固定根，不能悄悄切到当前 active。来源 pin 支持完整 SHA-1 / SHA-256。 + +仓库从固定视图的 `_repositories.yaml` 解析，旧目录只保留精确切片兼容，未知版本不猜测。 +跨仓库读取同时要求主机 `allowed_repositories` 授权与当前仓库中 `status: confirmed` +的依赖；目标源码和目录哈希必须匹配。工具参数只能选择目标，不能授予权限；所有目标 +共享同一个累计预算。 + +## 预算与正文 + +默认初始 24,000、最高 64,000；模型上下文默认配置 131,072，源码留白 32,000、输出 +留白 8,192，还可预留其他 prompt。可用上限取知识上限和模型剩余窗口的较小值。 +扩大必须给出原因，不能占用留白。这些是主机配置，不能冒称探测到 served model 窗口。 + +默认以 UTF-8 字节数作保守计量上界，`token_accounting` 明示此单位；主机可传稳定 ID +的 tokenizer。`actual_provider_usage` 保持 unknown。计量包括正文、来源字段、JSON +转义与 untrusted 围栏。仅 `model_content` 用于注入；`documents` 是不带额外正文的引用。 +片段截断清空完整已注入维度及对应 basis/mode/validation 标签；可用维度仍可补读。 + +SQLite 运行账本必须位于知识树之外。事务串行记账，精确来源单元去重，重复请求幂等； +扩大后允许继续未交付片段。缓存命中复核请求收据、正文摘要、计量和源文件；这属于完整性 +检查，不是针对可任意改写整个主机账本的签名。搜索与阅读重叠片段不保证全局语义去重。 + +## 接入与验证 + +新版 SDK 显式 `plan_adaptive` / 会话接口使用本服务；旧 `plan` 保留两页、6,000 字符契约。 +MCP 暴露同一 open/read/search/related/expand；Strict 在注册表快照默认 adaptive，旧树 +默认 legacy。规则与 briefing 经 `inject`，只接收真实文档 ID，不接收任意宿主散文。 + +`test_knowledge_context.py`、`test_knowledge_retrieval.py` 验证累计预算、并发、恢复、 +留白、跨仓库权限、缓存篡改、实际注入标签以及 SDK/Strict 的固定快照接入。 diff --git a/doc/architecture/SPEC/knowledge_docs.md b/doc/architecture/SPEC/knowledge_docs.md index aa590faf9..c106c93b0 100644 --- a/doc/architecture/SPEC/knowledge_docs.md +++ b/doc/architecture/SPEC/knowledge_docs.md @@ -1,6 +1,6 @@ # knowledge_docs.py —— 规范 - + `LOC ~280 · 供 Direct 与 Agent 共用的只读知识检索 · refactor-status: ok` @@ -33,7 +33,8 @@ stdlib 与 `knowledge_service.lifecycle` 的页面/完整深读正文解析； 结构化接口卡不进入背景。确定性优先级为固定来源路径、生产入口、源码 glob，再结合 功能 ID/标题和正文词命中；相同 feature 去重，完整深读优先于基础功能页。 最多返回两页、每页 3,000 字符、总计 6,000 字符；优先完整 facet，超长单 facet 仍提供 -截断片段并标 `more_available`。返回来源 pin、命中路径、已有/缺失 facet，不验证 PR head。 +截断片段并标 `partial` 与 `more_available`，不把截断 facet 列入 `included_facets`。 +返回来源 pin、命中路径、已有/缺失 facet，不验证 PR head。 `available_facets` 表示页内可用维度，`included_facets` 表示本次正文实际注入的维度； `not_injected_facets` 是已有但因预算未注入的维度，调用方可用原有有界读取补读。 `facet_basis` 与 `included_facet_basis` 分别对应已有及已注入维度的认可依据。 @@ -59,3 +60,12 @@ stdlib 与 `knowledge_service.lifecycle` 的页面/完整深读正文解析； ## 2026-09-28 退役规则不再提供 `read`/`search` 经 `visible_text` 去掉退役规则后再返回。 + +## 自适应调用边界（2026-10-06） + +旧默认仍是两页、6,000 字符。`related` 允许显式覆盖 `max_documents`、 +`max_page_chars`、`max_content_chars` 与 `max_changed_files`；自适应服务可取消改变文件 +列表的前缀限制。可选上限是 100 页、每页 65,536 字符、总 262,144 字符，最终实际交付 +仍由 `KnowledgeContextService` 累计预算裁剪。这些是字符限制，不是模型实际 token 用量。 +纯解析缓存按原文内容寻址，所有命中仍先验证固定快照的真实字节；不缓存路径读取以绕过 +完整性检查。此读取器没有累计补读账本；累计预算由显式会话服务负责。 diff --git a/doc/architecture/SPEC/sdk.md b/doc/architecture/SPEC/sdk.md index b833acc8d..3dc6ba5e0 100644 --- a/doc/architecture/SPEC/sdk.md +++ b/doc/architecture/SPEC/sdk.md @@ -1,6 +1,6 @@ # sdk/ —— 规范 - + `Python SDK v1 · 跨仓库唯一 typed 边界 · refactor-status: ok` @@ -17,6 +17,18 @@ ## 公开契约 +- 显式自适应知识协议：`ContextBudget`、`DirectClient.open_knowledge_context` / + `plan_adaptive` / `read_knowledge_context` / `search_knowledge_context` / + `related_knowledge_context` / `expand_knowledge_context`。仅 `model_content` 是实际注入 + 正文；引用没有 SDK 额外 excerpt。会话默认 24k、最大累计 64k，计量单位和实际用量 + 未知状态明确返回；规则、导航、相关知识和补读同预算。旧 v1 `plan` 默认两页/6k 不变。 +- 自适应会话支持 SHA-1/SHA-256 pin，可从私有运行 SQLite 恢复原冻结 view；签发的 + review context 绑定完整请求。旧 v1 completion 的 SHA-1 校验不因新增知识协议放宽。 + `allowed_knowledge_repositories` 是 DirectClient 构造时的主机权限配置，不是工具授权。 +- capabilities 的支持仓库来自固定视图 registry；新仓库不要求打包 adapter。 + `StrictRuntimeConfig` 在 registry 已识别仓库时允许不填 forge full_name；这只提供 + 知识身份兼容，不承诺旧 Strict PR 获取链路支持所有 forge 或无 forge PR。 + - `get_capabilities()` / `DirectClient.capabilities()` 返回 typed `Capabilities`：distribution/SDK/Direct/Strict/Quality/Knowledge 版本、resource revision、支持仓库、 expected-head/structured-result/post-false/file-lock/idempotency 能力与 worker 上限。 @@ -88,10 +100,10 @@ validator 缺失则写前 fail closed，执行失败/超时则逐 byte rollback 全部目标页。 同一 work checkout 的 writer 以 process 内 mutex 与位于系统临时目录的 `flock` 串行化；等待后的 SHA 复核让第二个 stale writer 失败，不会覆盖首个结果。 -- **知识 orchestration 留在宿主**：SDK 不 clone、调用 model、管理 ledger、commit、 +- **知识出版 orchestration 留在宿主**：SDK 不 clone、调用 model、管理出版 ledger、commit、 push、开 PR 或 schedule。ReviewBot 必须向 `KnowledgeCurator` 传 dedicated work checkout，并继续拥有重试、artifact 与本地补丁导出；SDK 也绝不写 packaged - knowledge tree。 + knowledge tree。自适应 DirectClient 仅管理知识上下文的运行账本，不代替知识出版治理。 - SDK、Quality API 版本常量为 `1.0.0`；Direct 为 `1.1.0`，Strict 为 `1.4.0` （1.1 新增 `finding_dispositions`；1.2 新增 `findings`；1.3 新增显式 recheck； 1.4 新增 `StrictRuntimeConfig.max_workers`），Knowledge 为 `1.1.0` diff --git a/doc/architecture/SPEC/thin_mcp_server.md b/doc/architecture/SPEC/thin_mcp_server.md index 5f7ffb517..c6967ca4e 100644 --- a/doc/architecture/SPEC/thin_mcp_server.md +++ b/doc/architecture/SPEC/thin_mcp_server.md @@ -1,6 +1,6 @@ # thin_mcp_server.py —— 规范 - + `LOC ~490 · 默认 MCP：Direct 门面 + Strict 入口 · refactor-status: ok` @@ -9,12 +9,17 @@ 并在被要求时桥接到 Strict。 ## 功能 -七个工具：`review`（按 `mode` 分流）、`validate_direct_review`、 +十二个工具：`review`（按 `mode` 分流）、`validate_direct_review`、 `get_review_status` / `get_review_result`（转发给 `CopilotMCP`）、 -`update_knowledge`、`doc_search`、`doc_read`。 +`update_knowledge`、`doc_search`、`doc_read`，以及显式自适应会话 +`open_knowledge_context` / `read_knowledge_context` / `search_knowledge_context` / +`related_knowledge_context` / `expand_knowledge_context`。 ## 公开契约 -上述七个工具；`build_mcp(...)`；`main()`。 +上述十二个工具；`build_mcp(...)`；`main()`。`review(knowledge_profile="adaptive")` +返回单一实际 `model_content` 和不重复正文的引用；默认 `legacy` 保持旧契约。 +会话工具累积预算，跨仓库目标只在主机授权与 confirmed 固定依赖同时成立时可读； +模型不能通过工具参数扩大权限。模型容量与源码/输出留白是明确配置，非探测结果。 ## 不变量（**C1**、**C2**、**D1**） - **Direct 在这个 server 里不跑任何模型。** 它返回知识路由和一份治理契约；阅读由 @@ -60,7 +65,8 @@ - **每个工具都声明 `ToolAnnotations`，且提示必须真实。** 审批门控的宿主（codex 对 无注解工具逐次弹批准框，headless 下自动取消，见 #86）靠这些提示放行只读面： `review` 是唯一保留状态变更（预留 Strict run）与触网（Strict 子进程）的工具， - 其余六个全部 `readOnlyHint=true`。把一个会写的工具标成只读，比不标更糟。 + 其余工具全部 `readOnlyHint=true`，相对于源码、知识内容和 forge 都只读；会话接口 + 会写主机私有运行账本。把一个会修改知识或源码的工具标成只读，比不标更糟。 （要求 `mcp>=1.8`，注解类型自该版本起可用。） ## 边界 —— 不属于这里 diff --git a/doc/architecture/feature-discovery.md b/doc/architecture/feature-discovery.md new file mode 100644 index 000000000..4365c7495 --- /dev/null +++ b/doc/architecture/feature-discovery.md @@ -0,0 +1,62 @@ +# 通用功能发现与目录冻结 + + + +功能发现位于 Skeleton 和 Breadth 之间。仓库适配配置确定扫描范围，源码 +commit、范围及库存索引固定后，先阅读已有文档正文，再扫描全部声明范围的 +源码和测试。源码轮优先处理原目录没有关联的区域，不限于文档关键词。 + +## 入口和契约单元 + +共享库存派生轻量定位单元：Python 的公开符号行段，以及其他语言中的导出、 +箭头函数、注册调用、路由、任务和 UI 事件线索。没有可识别入口的文件保留 +模块契约线索；未知语言、读取失败和解析失败不会使文件从库存中消失。 +这些单元是探索线索，一个函数、文件、配置项或页面不会自动成为一个功能。 +全文分片扫描仍然保留，单元分析不能代替完整库存处理。 + +单元补读包含实际入口和末端行段、同文件中直接提及的辅助定义、导入，以及 +包含该符号与断言的测试窗口。所有片段来自固定版本，遵守完整行及字符预算。 +这些关联只是定位信息，不构成运行时调用证明或测试覆盖证明；测试未执行。 + +## 有界补充发现 + +每个提取 packet 最多返回 24 个候选。达到上限后，将实际 packet 确定性拆小 +继续扫描，最多拆分两层；无法拆分或仍满额时保留 `overflow_unknown`，不会 +通过重复采样相同 packet 寻求更多候选。 + +第一轮候选评审后，最多执行两轮残留补扫，针对没有已认可功能映射的入口与 +契约单元补读。第一轮与第二轮的上下文预算不同；如果第二轮没有新增证据， +保留未知原因而不重复调用。每轮计划、任务和终态保存于检查点，预算停止后 +恢复同一计划，不重复成功扫描。候选的初次评审与三次修正计数跨补扫保持。 + +只有独立评审认可且引用真实的候选能够解释单元。引用过同一文件、未知候选 +或宽泛的目录 glob 不会使该文件中另一个入口被算作已解释。正文引用覆盖多个 +入口时，只关联认可说明明确提及的公开符号；单入口引用可以直接关联该单元。 +这种关联用于安排补扫，不是完整能力证明。仍未解释的单元 +和失败分片随报告交付，不必无限重试，不能声称所有功能已经发现。范围外的 +疑似第一方代码只形成建议，自动发现不改变声明范围及覆盖目标。 + +## 正式目录与证据交接 + +新增功能必须具有实现证据、独立的行为或契约边界及有效的原生评审记录。 +聚合跨前端、后端和 SDK 的同一能力；别名、实现补充、子能力、共享组件、 +过期声明与未知单列。原正式功能 ID 不自动删除、合并或拆分。 + +目录仍经过独立边界及聚合审查，再生成目录 PR。合并后冻结 ID、owner、 +边界、路径与 catalog hash，下游阶段使用同一身份。功能数由本仓库决定， +七维分母为当前冻结目录的 `N × 7`；历史目录分母单独保存。 + +完整报告及紧凑报告包含可携带的 `evidence_bundles`：功能 ID、源码 commit、 +scope/index/catalog hash、真实引用行段与内容摘要、unit ID 及原生记录。 +只携带定位引用，不内联重复源码或模型正文，也不把发现认可当成深度认可。 +基础提取与深化可以优先重读这些引用，再补读缺失维度。原生输入、输出、 +工具过程和完整发现报告存于 Git 外档案，紧凑报告给出路径、哈希及限制。 + +增量初始化可以指定已冻结库存内的 `scan_paths`，优先包含改变的路径及调用方、 +测试和相关文档。该集合绑定批次身份，正文扫描和残留目标只处理这些路径， +完整库存及功能目录保留。报告用 `scan_mode: incremental` 区分，不把局部处理 +称为全仓库扫描；相关测试片段仍可从其余固定库存中补读。删除或范围外的路径 +必须由更新入口单独记录，不能静默纳入或从正式目录删除功能。 + +`complete` 表示全部声明扫描任务及有界补扫具有终态，不表示未知已解决。 +后续深度及生产覆盖门槛仍独立判断，不能删除未知功能或缩小分母提高比例。 diff --git a/doc/architecture/portable-kb.md b/doc/architecture/portable-kb.md new file mode 100644 index 000000000..dfacc01fe --- /dev/null +++ b/doc/architecture/portable-kb.md @@ -0,0 +1,94 @@ +# 通用仓库接入、功能发现与知识服务 + +本流程按仓库自己的范围形成组件、功能和验收分母。生产库存不依赖 JiuwenSwarm 的文件数、语言或功能清单；已有打包 adapter 与旧 SDK 接口继续运行。 + +```mermaid +flowchart TD + A[任意 Git 软件仓库] --> B[生成配置提案并明确接受范围\n固定源码 S0] + B --> C[阶段 1 Skeleton：组件骨架] + C --> D[文档轮：作者已描述的功能] + D --> E[源码轮：公共入口、契约及完整声明分片] + E --> F[聚合去重与独立 Codex 评审] + F --> G[最多两轮残留补查\n满额证据包拆分、未知留存] + G --> H[目录审查并冻结哈希\nPR/MR 或明确本地回执] + H --> I[阶段 2 Breadth：导航、基础知识及未知] + I --> J[阶段 3 Deepen：七维认可及最终验收] + J --> K[单一主存储发布 S1\n另一处为只读镜像] + K --> L[统一知识上下文服务\nSDK / MCP / Direct / Strict] + M[新增入口、契约、生命周期或配置行为] --> E + P[仓库配置：范围、文档、语言、owner、预算和目标] + P --> B + P --> E + P --> I + P --> J +``` + +## 配置、源码与审查身份 + +`GitSource` 支持本地 Git（包括无 origin）、bare 仓库和可克隆 Git URL，固定 SHA‑1 或 SHA‑256 提交。库存与导出读取真实 Git blob，不读取工作区脏文件，也不应用 `export-ignore`、`export-subst`。Git 子模块和符号链接不作为已读取的目标源码；跨仓库需要独立的固定身份。 + +`RepoSpec` 保存稳定 ID、别名、源码 pin、范围、文档和测试 globs、存储选择、覆盖目标和依赖关系。自动提案需要审查；范围外疑似代码形成建议，不自动扩大当前扫描。二进制资源单列；未知语言的文本实现保留语言未知、解析失败和关系未知。空或仅含文档字符串的 Python 包标记采用已有确定性规则，不进入实现覆盖分母。 + +`ForgeProvider` 把来源与托管平台分离，提供 GitHub、GitLab 和无 forge 实现。合并回执校验真实状态及 head，不把本地接受伪装成已合并 PR。通用 CLI 目前通过明确本地回执接受阶段；原 adapter 的 PR 发布流程继续保留。GitLab 的源码/合并抽象不代表所有旧 PR 评审入口已经支持 GitLab。 + +```bash +kb --state-dir /work/kb-state onboard /work/project --repo-id project --out /work/proposal.json +# 审阅并调整 proposal.json；spec_sha256 必须与规范化 spec 一致。 +kb --state-dir /work/kb-state repo receipt --proposal /work/proposal.json --source /work/project --out /work/scope-review.json +kb --state-dir /work/kb-state repo register --proposal /work/proposal.json --review-receipt /work/scope-review.json --source /work/project --knowledge-root /work/copilot/knowledge +kb --state-dir /work/kb-state init project --stage skeleton +kb --state-dir /work/kb-state repo review-stage project --stage skeleton --reviewer maintainer --out /work/skeleton-review.json +kb --state-dir /work/kb-state repo accept-stage project --stage skeleton --review-receipt /work/skeleton-review.json +kb --state-dir /work/kb-state init project --stage feature-discovery +``` + +`review-stage` 生成精确文件、源码、目录及输入身份的本地审查回执；维护者审阅预览后执行 `accept-stage` 才接受。依次对 feature-discovery、modules、knowledge、knowledge-deepen 重复这组审查操作。未接受目录不能进入下游；预览被修改、源码身份变化或审查之后夹带提交均阻止发布。 + +## 两轮发现与有界补查 + +共享库存记录生产代码、嵌套测试和文档的完整内容哈希、语言、读取状态、符号和直接导入。全文分片全部安排处理；入口与契约单元把实现与相关文档、测试共同提供给模型，不能替代全文遍历。 + +文档轮要求实际正文支持候选。目录和链接只是线索；尚未确认实现的声明保留未知。源码轮先探索未关联区域，再遍历其余声明范围；纯库使用公共导出与模块契约，服务使用路由和事件任务，前端使用操作及状态契约，插件使用注册和生命周期，未知语言使用明确标记的文本线索。 + +新增功能须具有独立行为、契约或生命周期和真实实现，并经独立模型认可边界。同一能力的 UI、SDK、API 和存储实现可以合并；现有正式 ID 不自动删除、拆分或合并。候选分类包括新增、实现补充、别名、子能力、共享组件、过期和未知。 + +初次发现后检查未解释入口、满额候选包、只被未知或拒绝候选引用的单元、失败分片和范围建议。最多两轮补查；24 项满额包拆分后读取，每个候选最多初次尝试加三次修正，补查与恢复不重置已经消耗的次数。整文件引用不会自动解释其中所有公共入口。 + +检查点绑定源码、范围、旧目录、索引版本、提示词和模型配置。所有已安排任务必须有终态，预算耗尽不能标记完成；单项失败可作为未知交付，全局库存损坏或源码不符阻止冻结。报告分开统计扫描状态、候选分类、未解释入口和失败；扫描完成不表示所有功能已被发现。 + +确认结果生成不可变 `EvidenceBundle`，绑定源码、目录、索引、真实文件哈希、正文行段及原生评审回执。Breadth 与 Deepen 先验证再读取这些行段，包括长文件后部的实现；它们不重新猜测功能位置。新 owner 由 modules 建立。 + +## 发布、镜像与验收 + +主存储可以是 Copilot 的 `knowledge/`，也可以是目标仓库的 `.infermatrix/knowledge/`。`--storage in_repo` 时注册的 `--knowledge-root` 必须选择后一位置；另一处使用导出的只读镜像，不能成为写入主存储。生成的 `.infermatrix/**` 和镜像材料不进入功能发现。 + +S0 是源码证据提交；S1 是明确接受的知识提交，二者分别记录。发布复核真实源码、原生认可记录、政策与完整库存、格式、哈希和检索可读性，先验证暂存快照再替换主存储。全量服务快照保留主存储内容；单仓库镜像只包含该仓库、共享 general 和必要元数据，并分别绑定镜像哈希与主快照哈希。已被手工修改或含未绑定文件的镜像不会被覆盖。 + +基础阶段可以发布已核验知识及未知，`tier=foundation`、`init_complete=false`。最终要求每个正式功能至少一项认可深度、七个维度分别 **>90%**、生产文件结构覆盖 **≥85%**。目录改变产生新 revision 和 `N × 7` 分母；不删除未知来提高比例，不把旧 553 格基线混入新分母。 + +```bash +kb --state-dir /work/kb-state repo publish project --snapshot-out /work/snapshots/project-final +kb --state-dir /work/kb-state mirror project --out /work/project/.infermatrix/mirror +kb --state-dir /work/kb-state activate --snapshot /work/snapshots/project-final +# 明确选择服务基础知识，仍报告初始化未完成： +kb --state-dir /work/kb-state repo publish project --partial-foundation --snapshot-out /work/snapshots/project-foundation +kb --state-dir /work/kb-state activate --snapshot /work/snapshots/project-foundation --allow-partial +``` + +默认激活要求真实绑定的最终验收记录；`--allow-partial` 允许基础快照，但不能把它叫作完成。页面哈希和本地接受回执不等同于新的模型认可或上游测试通过。知识验证中的“测试入口”只表示引用了测试、断言或指南；上游测试是否执行另列。 + +## 统一服务、预算与跨仓库 + +新知识会话固定快照、仓库、源码和目录/政策身份。SDK、MCP、Direct 与 Strict 共用 `KnowledgeContextService`；搜索、规则、阅读和补读都计入同一累计预算，只注入返回的 `model_content`。实际注入正文、哈希、来源、截断及剩余维度状态保存在 Git 外 SQLite 账本。 + +默认最多 24k，按理由扩展至累计 64k，并预留源码、回答和其他提示容量。默认计量是 UTF‑8 字节上界估计；可配置实际 tokenizer 并固定其身份。估计预算不冒称为实际服务模型 token 用量。旧 SDK 默认仍为两页、6000 字符，显式开启 adaptive 会话后使用新预算；新的已注册 Strict 快照使用统一服务。 + +跨仓库读取要求宿主允许列表与注册表已确认依赖同时满足，并核对目标 source/catalog pin。模型工具参数不能自行授权。跨仓库线索可以帮助定位，但未授权、版本不明的源码不作为已核实证据；所有仓库共享当前会话预算。旧 `DirectCompletion` 的 PR SHA 仍采用 SHA‑1；新知识 pin 支持两种 Git 对象格式。 + +## 增量、运行配置与追踪 + +`kb update project --to --dry-run` 按真实 Git diff 报告新增、修改、删除和受影响功能；只有自身知识/镜像提交变化时不启动发现。`--apply` 准备独立固定源码批次，输出新 state-dir；随后以该目录运行 `init --stage feature-discovery --from-existing`。这是新批次准备，不是自动完成知识更新。导入的旧正文和认可仍是 S0 历史输入，不自动计为 S1 认可；新能力经过目录接受后改变分母。 + +默认 GLM‑5.3 提取与独立 Codex 评审，无 Claude。无显式 USD 上限的 portable 阶段使用双订阅模式；显式预算仍由相应阶段验证。`KB_GENERATOR`、`KB_JUDGE` 和发现阶段的 `KB_DISCOVERY_*` 可覆盖模型配置；`KB_KNOWLEDGE_CONCURRENCY` 调整基础提取并发。默认深化为轻量认可，显式 `--acceptance-mode strict` 保留严格模式。全局模型调度最多 13，提取与评审共用槽位；`KB_INIT_GLOBAL_CONCURRENCY` 可设为 1–13，`KB_INIT_DISPATCH_DIR` 指定共享锁目录（默认宿主缓存目录）。同一调度目录的上限必须一致；跨主机需要共享锁文件系统，不声称有分布式队列。 + +完整输入、输出、工具事件、配置和用量存于 state-dir 下的 `init/traces/` 与发现归档；Git 中只保存紧凑目录和报告、引用及摘要。没有账单费用时标为未知。本实现验证流程、隔离和恢复，不声称已实测任意仓库的发现召回率或 GLM PR 评审准确率提升。 diff --git a/doc/architecture/templates/kb-init-new-repository.yaml b/doc/architecture/templates/kb-init-new-repository.yaml index 2d1ffee8f..cd6087f5b 100644 --- a/doc/architecture/templates/kb-init-new-repository.yaml +++ b/doc/architecture/templates/kb-init-new-repository.yaml @@ -1,4 +1,6 @@ # Merge this snippet into a NEW repository adapter's manifest.yaml. +# Adapter-free repositories first use `kb onboard` and explicitly accept the +# generated scope proposal; see ../portable-kb.md for the full local workflow. # Retain the adapter's own knowledge_lifecycle mode, permissions and release policy. # Existing adapters opt in explicitly; this template does not migrate them. knowledge_lifecycle: diff --git a/eval/knowledge-depth/audit_depth_retrieval.py b/eval/knowledge-depth/audit_depth_retrieval.py index 28d0470bf..2c1789371 100644 --- a/eval/knowledge-depth/audit_depth_retrieval.py +++ b/eval/knowledge-depth/audit_depth_retrieval.py @@ -145,6 +145,17 @@ def _document(document: dict, view: KnowledgeView, pin: str, problems: list[str] problems.append(f"{label}: document exceeds 3000 characters") if any(row["pin"] != pin for row in sections.values()): problems.append(f"{label}: served depth has another source pin") + # A clipped snippet truthfully advertises no *intact* included facet. Bind + # its actual prose to one frozen section instead of inferring absence from + # included_facets=[] or accepting the partial flag without reading content. + fragment = content.rstrip() + partial_candidates = [facet for facet, section in sections.items() + if document.get("partial") is True and fragment.strip() + and len(fragment) < len(section["content"].strip()) + and section["content"].strip().startswith(fragment)] + partial_facet = partial_candidates[0] if len(partial_candidates) == 1 else None + if len(partial_candidates) > 1: + problems.append(f"{label}: partial context cannot be attributed to one intact facet") facet_states = {} for facet in FACETS: section = sections.get(facet) @@ -156,6 +167,8 @@ def _document(document: dict, view: KnowledgeView, pin: str, problems: list[str] injection = "partial" else: problems.append(f"{label}: declared injected facet {facet} is absent from actual content") + elif facet == partial_facet: + injection = "partial" facet_states[facet] = {"status": section["basis"] if section else "unknown", "injection": injection, "gap_label": section["gap_label"] if section else None} return {**document, "content_sha256": _hash(content), "page_sha256": _hash(raw), diff --git a/knowledge/tools/check_knowledge_tree.py b/knowledge/tools/check_knowledge_tree.py index 0acdbaa99..c03e74c63 100644 --- a/knowledge/tools/check_knowledge_tree.py +++ b/knowledge/tools/check_knowledge_tree.py @@ -117,7 +117,7 @@ def local_target(raw_target: str) -> str | None: target = target[1:-1] elif " " in target: target = target.split(" ", 1)[0] - if target.startswith(("http://", "https://", "mailto:", "#")): + if target.startswith(("http://", "https://", "repo://", "mailto:", "#")): return None return unquote(target.split("#", 1)[0]) diff --git a/knowledge/tools/check_wiki_lint.py b/knowledge/tools/check_wiki_lint.py index ea752c1ad..4fa0e2ede 100644 --- a/knowledge/tools/check_wiki_lint.py +++ b/knowledge/tools/check_wiki_lint.py @@ -173,7 +173,7 @@ def inbound_links() -> set[Path]: continue for m in MARKDOWN_LINK.finditer(line): t = m.group(1).split("#", 1)[0].strip() - if t and not t.startswith(("http://", "https://", "mailto:")): + if t and not t.startswith(("http://", "https://", "repo://", "mailto:")): targets.add((p.parent / t).resolve()) return targets diff --git a/src/infermatrix_copilot/app/run_service.py b/src/infermatrix_copilot/app/run_service.py index 720e93006..981a4e2fa 100644 --- a/src/infermatrix_copilot/app/run_service.py +++ b/src/infermatrix_copilot/app/run_service.py @@ -231,6 +231,7 @@ def _launch(self, run_id: str, *, strict_compat: bool = False) -> None: env["DEFAULT_REPO"] = self.settings.default_repo env["REPO_PATHS"] = json.dumps(self.settings.repo_paths) env["REPO_FULL_NAMES"] = json.dumps(self.settings.repo_full_names) + env["ALLOWED_KNOWLEDGE_REPOSITORIES"] = json.dumps(self.settings.allowed_knowledge_repositories) # The child re-authorizes the request's `repo_path`, so it must judge it # against the SAME roots and identities this server did — otherwise the # re-check silently validates against different config than the parent. @@ -557,17 +558,30 @@ def list_playbooks(self) -> dict: def _docs(self, repo: str) -> KnowledgeDocs: """Build the same repo-scoped knowledge view used by workflow agents.""" from ..adapters.base import AdapterRegistry + from ..knowledge_view import KnowledgeView, _load_view + from ..kb_service.repo_spec import resolve_snapshot_repo repo = repo or self.settings.default_repo - if repo not in self.settings.mcp_allowed_repos: + current = KnowledgeView.current() + kdir = Path(self.settings.knowledge_dir).resolve() + if current.verified or current.root.resolve() == kdir: + view = current + elif (kdir.parent / "MANIFEST.json").is_file(): + view = _load_view(str(kdir.parent)) + else: + view = KnowledgeView(kdir, f"unverified:{kdir}") + binding = resolve_snapshot_repo(view, repo) + if repo not in self.settings.mcp_allowed_repos and (binding is None or binding.repo_id not in self.settings.mcp_allowed_repos): raise PolicyError(f"repo {repo!r} is not permitted") + if binding is not None: + return KnowledgeDocs(view.root, binding.knowledge_slice, verify=view.path) try: adapter = AdapterRegistry(self.settings.adapters_dir).resolve( name=repo.replace("-", "_")) except Exception as exc: raise PolicyError(f"no knowledge adapter for repo {repo!r}") from exc kn = adapter.manifest.get("knowledge") or {} - return KnowledgeDocs(self.settings.knowledge_dir, kn.get("repo_subdir")) + return KnowledgeDocs(view.root, kn.get("repo_subdir"), verify=view.path) def doc_search(self, query: str, repo: str = "", limit: int = 40) -> dict: """Search general + the selected repo's curated Markdown knowledge.""" diff --git a/src/infermatrix_copilot/config.py b/src/infermatrix_copilot/config.py index ac86fc3e0..8a87437ee 100644 --- a/src/infermatrix_copilot/config.py +++ b/src/infermatrix_copilot/config.py @@ -201,6 +201,8 @@ class Settings(BaseSettings): # repos. knowledge_general_docs is the always-on general slice; every deeper # doc is reachable on demand via doc_search/doc_read. knowledge_dir: Path = _resource_dir("knowledge") + # Host authorization only: model tools cannot enlarge their repository scope. + allowed_knowledge_repositories: tuple[str, ...] = () knowledge_general_docs: list[str] = ["general/_index.md"] # Drop directory for bugfix-run intake records (KNOWLEDGE_INTAKE_DIR): # after a pr_debug run lands a real push, `pr.harvest_debug_knowledge` diff --git a/src/infermatrix_copilot/direct_routing.py b/src/infermatrix_copilot/direct_routing.py index 4a4977c8c..b44098574 100644 --- a/src/infermatrix_copilot/direct_routing.py +++ b/src/infermatrix_copilot/direct_routing.py @@ -78,7 +78,7 @@ def _adapter_repo_name(adapter: RepoAdapter) -> str: def load_routes(repo: str, view: KnowledgeView | None = None) -> dict | None: """Return a repository's validated `_routes.yaml`, or None when it has none.""" view = _view(view) - return _load_routes(view, _normalize_repo(repo)) + return _load_routes(view, _normalize_repo(repo, view)) _REPO_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") @@ -198,10 +198,14 @@ def _validate_routes(data: object, relative: str, view: KnowledgeView) -> dict: _EVIDENCE_HEAD_SHA = re.compile(r"[0-9a-f]{7,40}") -def _normalize_repo(repo: str) -> str: +def _normalize_repo(repo: str, view: KnowledgeView | None = None) -> str: selected = str(repo or "").strip() if not selected: raise ValueError("repo is required") + from .kb_service.repo_spec import resolve_snapshot_repo + binding = resolve_snapshot_repo(_view(view), selected) + if binding is not None: + return binding.knowledge_slice.removeprefix("repos/") alias = _repo_aliases().get(selected.casefold()) if alias: return alias @@ -221,7 +225,7 @@ def _adapter_changed_file_routes( view: KnowledgeView | None = None, ) -> tuple[list[dict[str, object]], list[str]]: view = _view(view) - selected_repo = _normalize_repo(repo) + selected_repo = _normalize_repo(repo, view) adapter = _adapter_for_repo(selected_repo) if adapter is None: return [], [str(path).replace("\\", "/") for path in changed_files] @@ -425,7 +429,7 @@ def _direct_knowledge_routes( option: the owners were already computed and then discarded. """ view = _view(view) - selected_repo = _normalize_repo(repo) + selected_repo = _normalize_repo(repo, view) changed_files = changed_files or [] if not isinstance(changed_files, list) or any( not isinstance(path, str) for path in changed_files @@ -438,6 +442,13 @@ def _direct_knowledge_routes( # served this repo's owner knowledge for a repo we do not serve. routes_table = _load_routes(view, selected_repo) if routes_table is None or not routes_table["owners"]: + from .kb_service.repo_spec import resolve_snapshot_repo + registered = resolve_snapshot_repo(view, repo) + if registered is not None and registered.registry_hash: + return {"status": "description_unrouted", "selected_by": "snapshot_registry", + "changed_files_role": "unmatched_visible", "routes": [], "scope_validation": [], + "unmatched_changed_files": list(changed_files), + "unmatched_policy": "This registered repository has no owner route for these paths; feature context remains available."} # Adapter presence decides which non-default-repo case this is. Without the # gate, a repo we do not serve at all fell into the adapter path, whose # helper returns every changed file as "unmatched" when there is no @@ -631,7 +642,7 @@ def direct_review_plan( ] mandatory_review_guides = _direct_mandatory_review_guides(view) related_started = time.perf_counter() - normalized = _normalize_repo(repo) + normalized = _normalize_repo(repo, view) subdir = f"repos/{normalized}" if _REPO_NAME.fullmatch(normalized) and routing.get("status") != "unsupported_exact_router" else None related = KnowledgeDocs(view.root, subdir, verify=view.path).related( changed_files, query="\n".join((title, body, diff))) diff --git a/src/infermatrix_copilot/engine/agent_runtime/knowledge.py b/src/infermatrix_copilot/engine/agent_runtime/knowledge.py index 4cc2c5d82..ff74ef909 100644 --- a/src/infermatrix_copilot/engine/agent_runtime/knowledge.py +++ b/src/infermatrix_copilot/engine/agent_runtime/knowledge.py @@ -263,12 +263,40 @@ def _repo_docs_tool(ctx: StepContext, adapter) -> dict[str, ToolDef]: repo_subdir = None if adapter is not None: repo_subdir = (adapter.manifest.get("knowledge") or {}).get("repo_subdir") - docs = KnowledgeDocs(kdir, repo_subdir) - - def doc_search(query: str, **_: Any) -> str: + from ...knowledge_view import KnowledgeView, _load_view + from ...kb_service.repo_spec import resolve_snapshot_repo + current = KnowledgeView.current() + if current.root.resolve() == kdir.resolve(): + view = current + elif (kdir.parent / "MANIFEST.json").is_file(): + view = _load_view(str(kdir.parent.resolve())) + else: + view = KnowledgeView(kdir.resolve(), f"unverified:{kdir.resolve()}") + state = getattr(ctx, "state", {}) + params = getattr(ctx, "params", {}) + selector = str((state.get("task_spec") or {}).get("repo") or "") + binding = resolve_snapshot_repo(view, selector) if selector else None + if binding is not None: + repo_subdir = binding.knowledge_slice + docs = KnowledgeDocs(kdir, repo_subdir, verify=view.path) + profile = params.get("knowledge_context_profile", "adaptive" if (kdir / "_repositories.yaml").is_file() else "legacy") + adaptive = None + session_id = "" + if profile == "adaptive" and binding is not None: + from ...knowledge_context import ContextBudget, KnowledgeContextService + adaptive = KnowledgeContextService(view, ctx.run_dir / "knowledge-context.sqlite", + allowed_repositories=getattr(ctx.settings, "allowed_knowledge_repositories", ())) + head = str(state.get("pr_head_sha") or (state.get("task_spec") or {}).get("expected_head_sha") or "") + budget = ContextBudget(**dict(params.get("knowledge_context_budget") or {})) + session_id = adaptive.open_session(selector, source_pin=head, + review_id=ctx.run_dir.name, budget=budget)["session_id"] + + def doc_search(query: str, repository: str | None = None, **_: Any) -> str: """Tool: grep the knowledge base's markdown for `query`; return knowledge-relative `path:line:text` matches (capped), or a sentinel.""" try: + if adaptive is not None: + return json.dumps(adaptive.search(session_id, query, repository=repository), ensure_ascii=False) hits = docs.search(query) except KnowledgeDocsError as exc: return f"refused: {exc}" @@ -276,10 +304,12 @@ def doc_search(query: str, **_: Any) -> str: f"{h['path']}:{h['line']}:{h['text']}" for h in hits ) or "(no matching docs)" - def doc_read(path: str, offset: int = 0, **_: Any) -> str: + def doc_read(path: str, offset: int = 0, repository: str | None = None, **_: Any) -> str: """Tool: read a knowledge-base doc by its knowledge-relative path (windowed 24k chars; page with offset). Refuses paths escaping the base.""" try: + if adaptive is not None: + return json.dumps(adaptive.read(session_id, path, offset=offset, repository=repository), ensure_ascii=False) page = docs.read(path, offset=offset) except FileNotFoundError: return f"(no such doc: {path})" @@ -291,33 +321,57 @@ def doc_read(path: str, offset: int = 0, **_: Any) -> str: f"offset={page['next_offset']}]") return window - def doc_related(changed_files: list[str], query: str = "", **_: Any) -> str: + def doc_related(changed_files: list[str], query: str = "", repository: str | None = None, **_: Any) -> str: """The same bounded implementation context used by Direct plans.""" try: + if adaptive is not None: + packet = adaptive.related(session_id, changed_files, query=query, repository=repository) + ctx.trace.record("review_knowledge_injected", session_id=session_id, + model_content_sha256=packet["model_content_sha256"], + model_content=packet["model_content"], documents=packet["documents"], + consumed_tokens=packet["consumed_tokens"], token_accounting=packet["token_accounting"]) + return json.dumps(packet, ensure_ascii=False) return json.dumps(docs.related(changed_files, query=query), ensure_ascii=False) except KnowledgeDocsError as exc: return json.dumps({"status": "refused", "reason": str(exc), "documents": []}) s = {"type": "string"} - return { + tools = { "doc_related": ToolDef( - "doc_related", "Retrieve at most two relevant implementation explanations for changed files. " + "doc_related", "Retrieve relevant implementation explanations within the active context budget. " "Includes pinned sources and missing facets; verify against the PR head.", - {"type": "object", "properties": {"changed_files": {"type": "array", "items": s}, "query": s}, + {"type": "object", "properties": {"changed_files": {"type": "array", "items": s}, "query": s, "repository": s}, "required": ["changed_files"]}, doc_related), "doc_search": ToolDef( "doc_search", "Search the repo's curated knowledge base (community docs) for a " "string; returns knowledge-relative path:line matches to open with " "doc_read.", - {"type": "object", "properties": {"query": s}, "required": ["query"]}, + {"type": "object", "properties": {"query": s, "repository": s}, "required": ["query"]}, doc_search), "doc_read": ToolDef( "doc_read", "Read a doc from the curated knowledge base by its knowledge-relative " "path (e.g. `repos//rules.md`, or a `general/...` guide it " "links to). Windowed; page with offset.", - {"type": "object", "properties": {"path": s, + {"type": "object", "properties": {"path": s, "repository": s, "offset": {"type": "integer"}}, "required": ["path"]}, doc_read), } + if adaptive is not None: + def context_briefing(**_: Any) -> str: + paths = list(ctx.settings.knowledge_general_docs or []) + if adapter is not None: + paths += list((adapter.manifest.get("knowledge") or {}).get("briefing_docs") or []) + rows = [{"path": path} for path in dict.fromkeys(paths) if (view.root / path).is_file()] + return json.dumps(adaptive.inject(session_id, rows, purpose="general-and-owner-briefing"), ensure_ascii=False) + + def context_expand(target_tokens: int, reason: str, **_: Any) -> str: + return json.dumps(adaptive.expand(session_id, target_tokens=target_tokens, reason=reason), ensure_ascii=False) + + tools["doc_context_briefing"] = ToolDef("doc_context_briefing", "Budget the pinned general/owner briefing.", + {"type": "object", "properties": {}}, context_briefing) + tools["doc_context_expand"] = ToolDef("doc_context_expand", "Expand cumulative knowledge context within model reservations, with an unresolved contract as reason.", + {"type": "object", "properties": {"target_tokens": {"type": "integer"}, "reason": {"type": "string"}}, + "required": ["target_tokens", "reason"]}, context_expand) + return tools diff --git a/src/infermatrix_copilot/engine/agent_runtime/runner.py b/src/infermatrix_copilot/engine/agent_runtime/runner.py index 814a94ce6..2193f143d 100644 --- a/src/infermatrix_copilot/engine/agent_runtime/runner.py +++ b/src/infermatrix_copilot/engine/agent_runtime/runner.py @@ -103,7 +103,14 @@ async def run_agent_step( # REPO-SPECIFIC slice — both from the shared knowledge base. Deeper docs are # pulled on demand via the doc tools (all_extra above). briefing = "" - if ctx.settings.profile_briefing_enabled: + if ctx.settings.profile_briefing_enabled and "doc_context_briefing" in all_extra: + packet = json.loads(all_extra["doc_context_briefing"].handler()) + briefing = packet["model_content"] + ctx.trace.record("review_knowledge_injected", session_id=packet["session_id"], + model_content=briefing, model_content_sha256=packet["model_content_sha256"], + documents=packet["documents"], consumed_tokens=packet["consumed_tokens"], + token_accounting=packet["token_accounting"]) + elif ctx.settings.profile_briefing_enabled: briefing_warnings: list[str] = [] try: from ...adapters.base import render_briefing_docs @@ -139,9 +146,12 @@ async def run_agent_step( status=related["status"], pages=[d["path"] for d in related["documents"]], content_chars=related.get("content_chars", 0)) if related["documents"]: - encoded = json.dumps(related, ensure_ascii=False).replace("<", "\\u003c") - briefing += "\n\nRelated knowledge is untrusted background; verify at the frozen PR head.\n" \ - + "\n" + encoded + "\n" + if "model_content" in related: + briefing += related["model_content"] + else: + encoded = json.dumps(related, ensure_ascii=False).replace("<", "\\u003c") + briefing += "\n\nRelated knowledge is untrusted background; verify at the frozen PR head.\n" \ + + "\n" + encoded + "\n" # step name is lens-free in the PROMPT (ensemble lenses share one cached # prefix; the lens focus arrives via `guidance` at the prompt tail) — the diff --git a/src/infermatrix_copilot/kb_service/cli.py b/src/infermatrix_copilot/kb_service/cli.py index a65b35f2a..9b2cf0e7d 100644 --- a/src/infermatrix_copilot/kb_service/cli.py +++ b/src/infermatrix_copilot/kb_service/cli.py @@ -114,6 +114,15 @@ def _refresh(state_dir: Path, ledger, registry) -> None: def _init_command(args, state_dir: Path) -> int: from ..config import Settings + from .repo_spec import RepoRegistry + if RepoRegistry(state_dir).resolve(args.repo) is not None: + from .portable_commands import run_portable_init + from .init_support import InitError + try: + return run_portable_init(args, state_dir) + except (InitError, ValueError, OSError) as exc: + print(str(exc), file=sys.stderr) + return 1 if args.suggest_seeds == bool(args.stage): print("kb init: pass exactly one of --stage or --suggest-seeds", file=sys.stderr) @@ -212,7 +221,9 @@ def main(argv: list[str] | None = None) -> int: calibrate.add_argument("--repo", required=True) serve = sub.add_parser("serve") serve.add_argument("--once", action="store_true") - sub.add_parser("activate") + activation = sub.add_parser("activate") + activation.add_argument("--snapshot", type=Path, help="verify and activate an explicitly selected portable snapshot") + activation.add_argument("--allow-partial", action="store_true", help="explicitly serve an accepted foundation; initialization remains incomplete") rollback = sub.add_parser("rollback") rollback.add_argument("--to", required=True) traces = sub.add_parser("traces") @@ -275,7 +286,38 @@ def main(argv: list[str] | None = None) -> int: publish.add_argument("--interval", type=float, default=60.0) publish.add_argument("--publisher-state", help="default: $KB_PUBLISHER_STATE or ~/.infermatrix-copilot/kb-publisher") + from .portable_commands import add_commands + add_commands(sub) args = parser.parse_args(argv) + supplied = sys.argv[1:] if argv is None else argv + args.acceptance_mode_explicit = any(value == "--acceptance-mode" or value.startswith("--acceptance-mode=") for value in supplied) + + if args.command in {"onboard", "repo", "mirror", "update"}: + from .portable_commands import command + from .init_support import InitError + from .sources import SourceError + from .knowledge_store import KnowledgeStoreError + try: + return command(args, _state_dir(args.state_dir)) + except (InitError, SourceError, KnowledgeStoreError, ValueError, OSError, KeyError) as exc: + print(str(exc), file=sys.stderr) + return 1 + if args.command == "activate" and args.snapshot is not None: + from .activate import activation_lock, verify_snapshot, switch_active, ActivationError + from .portable_publication import check_publication + from .knowledge_store import KnowledgeStoreError + try: + snapshot = args.snapshot.resolve(strict=True) + state_dir = _state_dir(args.state_dir) + with activation_lock(state_dir): + view = verify_snapshot(snapshot) + publication = check_publication(view, allow_partial=args.allow_partial) + switch_active(state_dir, snapshot) + print(json.dumps({"active_snapshot": view.public_snapshot, "verified": True, **publication})) + return 0 + except (ActivationError, KnowledgeStoreError, OSError, ValueError) as exc: + print(str(exc), file=sys.stderr) + return 1 if args.command == "keygen": from ..knowledge_service.signing import generate_private_key, public_key_text diff --git a/src/infermatrix_copilot/kb_service/depth_index.py b/src/infermatrix_copilot/kb_service/depth_index.py index 776f5c829..7cdd2915a 100644 --- a/src/infermatrix_copilot/kb_service/depth_index.py +++ b/src/infermatrix_copilot/kb_service/depth_index.py @@ -55,7 +55,7 @@ def files(self): def _identity(pin, policy_sha256, production): - if not re.fullmatch(r"[a-f0-9]{40}", pin or ""): + if not re.fullmatch(r"(?:[a-f0-9]{40}|[a-f0-9]{64})", pin or ""): raise ValueError("shared depth index needs a full immutable pin") if not re.fullmatch(r"[a-f0-9]{64}", policy_sha256 or ""): raise ValueError("shared depth index needs the policy SHA256") diff --git a/src/infermatrix_copilot/kb_service/depth_inputs.py b/src/infermatrix_copilot/kb_service/depth_inputs.py index fd47640b8..61097c803 100644 --- a/src/infermatrix_copilot/kb_service/depth_inputs.py +++ b/src/infermatrix_copilot/kb_service/depth_inputs.py @@ -247,7 +247,7 @@ class DepthContext: """ def __init__(self, tree: Path, production: list[str], *, index=None, cache_path=None, - pin=None, policy_sha256=None, mode="strict"): + pin=None, policy_sha256=None, mode="strict", discovery=None): system_prompt(mode) if index is not None and cache_path is not None: raise ValueError("provide an index or cache_path, not both") @@ -256,6 +256,7 @@ def __init__(self, tree: Path, production: list[str], *, index=None, cache_path= index = load_depth_index(cache_path, pin=pin, policy_sha256=policy_sha256, production=production) self.tree, self.production = tree, sorted(set(production)) self.index, self.mode = index, mode + self.discovery = discovery or {} if index is not None: from .depth_index import _identity expected = _identity(pin or index.identity["pin"], policy_sha256 or index.identity["policy_sha256"], production) @@ -887,6 +888,13 @@ def build(self, feature, existing: str, docs: list[dict], *, facets=None, else: relevant.extend(p for p in self.production if self._import_targets(p)[0] & set(source)) relevant = list(dict.fromkeys(relevant)) + discovery_bundle = self.discovery.get("evidence_bundles", {}).get(feature.id) + discovery_spans = [] + if discovery_bundle: + from .evidence_bundle import materialize_bundle + discovery_spans = materialize_bundle(discovery_bundle, self.tree, pin=self.discovery["pin"], + catalog_hash=self.discovery["catalog_sha256"]) + relevant = list(dict.fromkeys([s["path"] for s in discovery_spans if s["kind"] != "doc"] + relevant)) tiers = {path: 3 if path in feature.entry_points else 2 if path in source else 1 if path in dependencies else 0 for path in relevant} @@ -972,6 +980,9 @@ def build(self, feature, existing: str, docs: list[dict], *, facets=None, flow_pairs.append((tiers[path], base + 35, pair)) ranges.extend((base + 35, p, start, end) for p, start, end in pair) # Sorting before rotation preserves priority; only ties rotate between rounds. + for span in discovery_spans: + if span["kind"] != "doc": + ranges.append((20_000, span["path"], span["start"], span["end"])) ordered = sorted(set(ranges), key=lambda r: (-tiers.get(r[1], 4), -r[0], r[1], r[2], r[3])) if evidence_round and ordered: rotated = [] @@ -1093,5 +1104,7 @@ def rank(item): payload["docs"] = self.document_slices(docs, paths=getattr(feature, "docs", ()), previous_review=previous_review, evidence_round=evidence_round, facets=facets, symbols=symbols) + if discovery_bundle: + payload["discovery_bundle_sha256"] = discovery_bundle["bundle_sha256"] payload["context_sha256"] = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest() return payload diff --git a/src/infermatrix_copilot/kb_service/evidence_bundle.py b/src/infermatrix_copilot/kb_service/evidence_bundle.py new file mode 100644 index 000000000..d9a93afcd --- /dev/null +++ b/src/infermatrix_copilot/kb_service/evidence_bundle.py @@ -0,0 +1,92 @@ +"""Portable discovery handoffs. References are localization evidence, not new approval.""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path, PurePosixPath + + +def digest(value): + return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(",", ":")).encode()).hexdigest() + + +def build_evidence_bundle(index, feature_id, *, catalog_hash, refs, unit_ids=(), receipts=()): + from .feature_discovery_index import validate_evidence + + if not isinstance(feature_id, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,39}", feature_id): + raise ValueError("invalid evidence bundle feature") + if not re.fullmatch(r"[0-9a-f]{64}", catalog_hash): + raise ValueError("evidence bundle requires frozen catalog hash") + selected = {} + for ref in refs: + if not validate_evidence(index, ref): + raise ValueError("discovery evidence reference is invalid") + entry = index.entries[ref["path"]] + item = {"path": ref["path"], "start": ref["start"], "end": ref["end"], + "sha256": entry["sha256"], "kind": entry["kind"]} + selected[(item["path"], item["start"], item["end"])] = item + body = {"schema_version": 1, "feature_id": feature_id, "pin": index.identity["pin"], + "scope_sha256": digest(index.identity["scope"]), "index_sha256": index.sha256, + "catalog_sha256": catalog_hash, "refs": [selected[k] for k in sorted(selected)], + "unit_ids": sorted(set(unit_ids)), "receipts": list(receipts)} + return {**body, "bundle_sha256": digest(body)} + + +def materialize_bundle(bundle, tree, *, pin, catalog_hash): + """Verify every referenced full-file hash before returning its exact numbered span.""" + if not isinstance(bundle, dict) or bundle.get("schema_version") != 1: + raise ValueError("unsupported discovery evidence bundle") + body = {k: v for k, v in bundle.items() if k != "bundle_sha256"} + if bundle.get("bundle_sha256") != digest(body): + raise ValueError("discovery evidence bundle hash mismatch") + if bundle.get("pin") != pin or bundle.get("catalog_sha256") != catalog_hash: + raise ValueError("discovery evidence bundle source/catalog mismatch") + root = Path(tree).resolve() + result = [] + for ref in bundle["refs"]: + path = PurePosixPath(ref["path"]) + if path.is_absolute() or ".." in path.parts or "\\" in ref["path"]: + raise ValueError("discovery evidence path escapes source") + target = root.joinpath(*path.parts) + if target.is_symlink() or not target.resolve().is_relative_to(root): + raise ValueError("discovery evidence path escapes source") + raw = target.read_bytes() + if hashlib.sha256(raw).hexdigest() != ref["sha256"]: + raise ValueError("discovery evidence source hash mismatch") + lines = raw.decode("utf-8").splitlines() + start, end = ref["start"], ref["end"] + if type(start) is not int or type(end) is not int or not 1 <= start <= end <= len(lines): + raise ValueError("discovery evidence span is invalid") + result.append({**ref, "total_lines": len(lines), + "text": "\n".join(f"{i}: {lines[i-1]}" for i in range(start, end + 1))}) + return result + + +def affected_features(bundles, changed_paths): + """Conservative path invalidation; unmodelled dynamic relations remain outside this proof.""" + changed = set(changed_paths) + return sorted(fid for fid, bundle in bundles.items() + if any(ref["path"] in changed for ref in bundle.get("refs", []))) + + +def bounded_spans(items, limit): + """Keep complete numbered lines, preserving the original non-prefix location.""" + result, used, seen = [], 0, set() + for item in items: + key = (item["path"], item.get("start", 1), item["end"]) + if key in seen: + continue + seen.add(key) + selected = [] + for line in item["text"].splitlines(): + size = len((line + "\n").encode()) + if used + size > limit: + break + selected.append(line) + used += size + if selected: + result.append({**item, "end": item.get("start", 1) + len(selected) - 1, + "text": "\n".join(selected)}) + return result diff --git a/src/infermatrix_copilot/kb_service/feature_discovery_index.py b/src/infermatrix_copilot/kb_service/feature_discovery_index.py index da2552e42..3b35a612c 100644 --- a/src/infermatrix_copilot/kb_service/feature_discovery_index.py +++ b/src/infermatrix_copilot/kb_service/feature_discovery_index.py @@ -17,6 +17,7 @@ import uuid INDEX_VERSION = "feature-discovery-index-v2" +CONTRACT_UNIT_VERSION = "contract-units-v1" MAX_CHUNK_CHARS = 12_000 DOC_SUFFIXES = {".md", ".mdx", ".rst", ".adoc", ".txt"} _RESOURCE_SUFFIXES = {".json", ".yaml", ".yml", ".toml", ".ini", ".lock", ".xml", ".csv", ".svg", ".map", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", @@ -158,7 +159,7 @@ def failed_directory(exc): def _identity(pin, scope, doc_globs, source_digest): - if not re.fullmatch(r"[a-f0-9]{40}", pin or ""): + if not re.fullmatch(r"(?:[a-f0-9]{40}|[a-f0-9]{64})", pin or ""): raise ValueError("discovery index needs a full immutable source SHA") roots = list(scope.get("roots") or (".",)) for root in roots: @@ -193,16 +194,21 @@ def build_discovery_index(tree, *, pin, scope, doc_globs=(), cache_path=None): except ValueError: pass # stale/corrupt caches never alter this batch's source scope roots, excluded = identity["scope"]["roots"], scope.get("exclude", ()) extensions, filenames = scope.get("suffixes"), scope.get("filenames", ()) + test_globs = tuple(scope.get("test_globs", ())) entries, production, tests, docs, failures, suggestions = {}, [], [], [], [], [] for path in names: if not _safe(path): raise ValueError("unsafe tracked path in discovery inventory") suffix = PurePosixPath(path).suffix.lower() doc = _match(path, doc_globs) - test = is_test(path) + test = is_test(path) or _match(path, test_globs) exclusion_hits = [pattern for pattern in excluded if _match(path, [pattern])] # Test directories excluded from production remain available to the # test index. Other exclusions (vendor/build/generated) remain effective. - if exclusion_hits and not (test and all(_test_only_exclusion(p) for p in exclusion_hits)): + def test_exclusion(pattern): + hard = {"vendor", "third_party", "third-party", "node_modules", "generated", "dist", "build"} + return _test_only_exclusion(pattern) or (pattern in test_globs and + not any(part in hard for part in pattern.casefold().split("/"))) + if exclusion_hits and not (test and all(test_exclusion(p) for p in exclusion_hits)): continue known = suffix in _LANGUAGES or _match(PurePosixPath(path).name, filenames) unfamiliar = suffix not in DOC_SUFFIXES | _RESOURCE_SUFFIXES and bool(suffix) and not known @@ -220,7 +226,7 @@ def build_discovery_index(tree, *, pin, scope, doc_globs=(), cache_path=None): source = _under(path, roots) and (PurePosixPath(path).suffix in extensions or _match(PurePosixPath(path).name, filenames)) if not doc and not test and not source: - if known and not _under(path, roots): suggestions.append({"path": path, "reason": "code outside declared production roots"}) + if (known or unfamiliar) and not _under(path, roots): suggestions.append({"path": path, "reason": "code outside declared production roots"}) elif (known or unfamiliar or shebang) and _under(path, roots): suggestions.append({"path": path, "reason": "unsupported suffix outside declared suffix scope"}) continue kind = "doc" if doc else "test" if test else "source" @@ -330,6 +336,128 @@ def evidence_excerpt(index, reference): "sha256": entry["sha256"], "kind": entry["kind"]} +def contract_units(index): + """Conservative public-entry/contract leads, never certified features. + + Unit identity is independent of the candidate/catalog set. Parsed Python + spans and lexical declarations/registrations in other languages supply + anchors; opaque or unreadable files retain an explicit module lead. The + complete inventory scan remains authoritative even when these hints miss + a dynamic registration or a language-specific construct. + """ + units = [] + declaration = re.compile( + r"\b(?:export\s+(?:default\s+)?(?:async\s+)?(?:function|class|const|let)|" + r"pub(?:\([^)]*\))?\s+(?:async\s+)?(?:fn|struct|trait)|" + r"(?:public|open)\s+(?:static\s+)?(?:fun|class|interface)|" + r"(?:function|class|interface|struct|fn|func|fun))\s+([\w$]+)") + arrow = re.compile(r"\b(?:export\s+)?(?:const|let|var)\s+([\w$]+)\s*=.*(?:=>|function\b)") + registration = re.compile( + r"(?:\.(?:get|post|put|delete|patch|route|command|add_parser|register|" + r"register_plugin|add_route|add_task|schedule)\s*\(|" + r"@(?:\w+\.)*(?:route|command|task)\b|" + r"\b(?:module\.exports|exports\.[\w$]+)\s*=)") + ui_event = re.compile(r"\b(?:onClick|onSubmit|onChange|on:click|on:submit)\s*=|@(?:click|submit)=") + for path in index.production: + entry = index.entries[path] + lines = entry.get("lines", []) + anchors = [] + if entry.get("status") == "ready": + if entry.get("language") == "python" and entry.get("parse_status") == "parsed": + anchors.extend((s["start"], s.get("end", s["start"]), "public_contract", s["name"]) + for s in entry.get("symbols", []) if not s["name"].startswith("_")) + for number, line in enumerate(lines, 1): + match = declaration.search(line) or arrow.search(line) + if match and entry.get("language") != "python": + anchors.append((number, min(len(lines), number + 39), "public_contract", match[1])) + if registration.search(line): + anchors.append((number, min(len(lines), number + 39), "registration", line.strip()[:120])) + if ui_event.search(line): + anchors.append((number, min(len(lines), number + 19), "ui_entry", line.strip()[:120])) + if not anchors: + anchors.append((1 if lines else 0, len(lines), "module_contract", PurePosixPath(path).stem)) + for start, end, kind, name in sorted(set(anchors)): + units.append({"id": _sha([CONTRACT_UNIT_VERSION, index.identity["pin"], path, + entry.get("sha256"), start, end, kind, name]), + "path": path, "start": start, "end": end, "kind": kind, + "name": name, "language": entry.get("language", "unknown"), + "status": entry.get("status", "error"), + "parse_status": entry.get("parse_status", "unknown"), + "sha256": entry.get("sha256", "")}) + return units + + +def unit_test_lookup(index, names): + """One shared pass over nested tests for all named entry leads.""" + wanted = {name for name in names if re.fullmatch(r"[A-Za-z_$][\w$]{2,}", name)} + found = {name: [] for name in wanted} + for path in index.tests: + test = index.entries[path] + if test.get("status") != "ready": + continue + seen = set() + for number, line in enumerate(test["lines"], 1): + hits = set(re.findall(r"[A-Za-z_$][\w$]*", line)) & wanted - seen + if not hits: + continue + window = test["lines"][max(0, number - 6):number + 10] + if re.search(r"\bassert\w*\b|\bexpect\s*\(|\braises\s*\(", "\n".join(window)): + ref = {"path": path, "start": max(1, number - 5), "end": min(len(test["lines"]), number + 10)} + for name in hits: + found[name].append(ref) + seen.update(hits) + return found + + +def unit_evidence(index, unit, *, max_chars=12000, test_lookup=None): + """Bounded anchor/body plus directly named imports/tests, all real lines. + + Related text is only a localization hint. No lexical match establishes a + runtime call, test association, executed test or verified absence. + """ + if type(max_chars) is not int or max_chars < 1: + raise ValueError("unit evidence budget must be positive") + entry = index.entries[unit["path"]] + if entry.get("status") != "ready" or unit["start"] < 1: + return [] + result, used = [], 0 + + def offer(path, start, end): + nonlocal used + item = evidence_excerpt(index, {"path": path, "start": start, "end": end}) + # Preserve whole lines. An oversize anchor stays an unresolved lead; + # partial text must never be offered with a complete-line citation. + while len(item["text"]) + used + 150 > max_chars and end > start: + end -= 1 + item = evidence_excerpt(index, {"path": path, "start": start, "end": end}) + if len(item["text"]) + used + 150 > max_chars: + return + if any(p["path"] == path and p["start"] <= start and p["end"] >= end for p in result): + return + result.append(item) + used += len(item["text"]) + 150 + + start, end = unit["start"], unit["end"] + offer(unit["path"], max(1, start - 3), min(end, start + 59)) + if end > start + 59: + offer(unit["path"], max(start + 60, end - 19), end) + offered_body = "\n".join(item["text"] for item in result) + helpers = [s for s in entry.get("symbols", []) if s["start"] != start and + re.search(r"(?= max_chars - 150: + break + return result + + def build_for_stage(tree, stage): from .knowledge_coverage import load_policy, policy_path policy = getattr(stage, "coverage_policy", None) @@ -340,6 +468,12 @@ def build_for_stage(tree, stage): doc_globs = list(stage.lifecycle.init.doc_globs) if policy: doc_globs += list(policy.catalog_sources) scope = discovery_scope(stage.lifecycle.init, policy) + manifest = getattr(stage, "manifest", {}) or {} + portable = manifest.get("portable_scope") if isinstance(manifest, dict) else None + if isinstance(portable, dict): + if policy is None: + scope.update(suffixes=list(portable.get("suffixes", [])), filenames=list(portable.get("filenames", []))) + scope["test_globs"] = list(portable.get("test_globs", [])) name = _sha([stage.record.pin, scope, sorted(set(doc_globs))]) return build_discovery_index(tree, pin=stage.record.pin, scope=scope, doc_globs=sorted(set(doc_globs)), cache_path=stage.rt.state_dir / "feature-discovery-index" / f"{name}.json") diff --git a/src/infermatrix_copilot/kb_service/git_source.py b/src/infermatrix_copilot/kb_service/git_source.py new file mode 100644 index 000000000..f9595d77b --- /dev/null +++ b/src/infermatrix_copilot/kb_service/git_source.py @@ -0,0 +1,277 @@ +"""Committed Git trees, independent of hosting and review providers.""" + +from __future__ import annotations + +import json +import re +import subprocess +import urllib.parse +import urllib.request +from pathlib import Path, PurePosixPath +from typing import Callable + +from .sources import SourceError + +COMMIT_ID = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") + + +def public_locator(locator: str) -> str: + """Remove HTTP credentials, queries and fragments from a public identity.""" + value = str(locator).strip() + if re.match(r"^[A-Za-z]:[\\/]", value): + return "" + if "://" in value: + url = urllib.parse.urlsplit(value) + if url.scheme not in ("http", "https", "ssh", "git", "file"): + raise SourceError("unsupported Git locator scheme") + if url.scheme == "file": + return "" # host paths belong to runtime bindings + host = url.hostname or "" + if not host: + raise SourceError("Git locator has no host") + port = f":{url.port}" if url.port else "" + return urllib.parse.urlunsplit((url.scheme, host.lower() + port, url.path.rstrip("/"), "", "")) + match = re.fullmatch(r"(?:[^/@:\s]+@)?([^/:\s]+):(.+)", value) + if match: + return f"ssh://{match[1].lower()}/{match[2].rstrip('/')}" + return "" # ordinary local paths are never public locators + + +class GitSource: + """Read the fixed committed tree of a local normal or bare repository. + + Neither dirty working files nor an ``origin`` are required or consulted + when reading evidence. Git's own object format determines the pin width. + """ + + def __init__(self, path: str | Path): + self.path = Path(path).expanduser().resolve() + self._inventories: dict[str, dict] = {} + self._git("rev-parse", "--git-dir") + + @property + def git_dir(self) -> Path: + value = self._git("rev-parse", "--absolute-git-dir").decode().strip() + return Path(value) + + @classmethod + def acquire(cls, locator: str | Path, cachepath: str | Path) -> "GitSource": + """Make a bare source mirror; local repositories need no origin.""" + target = Path(cachepath).resolve() + if target.exists(): + raise SourceError("source acquisition destination already exists") + target.parent.mkdir(parents=True, exist_ok=True) + proc = subprocess.run(["git", "clone", "--quiet", "--mirror", "--", str(locator), str(target)], + capture_output=True, check=False) + if proc.returncode: + raise SourceError(f"Git source acquisition failed (exit {proc.returncode})") + return cls(target) + + def _git(self, *args: str) -> bytes: + proc = subprocess.run(["git", "-C", str(self.path), *args], capture_output=True, check=False) + if proc.returncode: + # stderr can contain credential-bearing remote URLs. + raise SourceError(f"Git source operation {args[0]} failed (exit {proc.returncode})") + return proc.stdout + + @property + def object_format(self) -> str: + value = self._git("rev-parse", "--show-object-format").decode().strip() + if value not in ("sha1", "sha256"): + raise SourceError("unsupported Git object format") + return value + + def resolve(self, ref: str = "HEAD") -> str: + if not isinstance(ref, str) or not ref or ref.startswith("-") or "\0" in ref: + raise SourceError("invalid Git revision") + pin = self._git("rev-parse", "--verify", f"{ref}^{{commit}}").decode().strip() + width = 40 if self.object_format == "sha1" else 64 + if not COMMIT_ID.fullmatch(pin) or len(pin) != width: + raise SourceError("revision is not a full commit identity") + return pin + + def head(self) -> str: + return self.resolve() + + def origin(self) -> str: + try: + return public_locator(self._git("remote", "get-url", "origin").decode().strip()) + except SourceError: + return "" + + def root_commits(self, pin: str) -> tuple[str, ...]: + return tuple(sorted(self._git("rev-list", "--max-parents=0", self.resolve(pin)).decode().split())) + + def tree_id(self, pin: str) -> str: + return self._git("rev-parse", f"{self.resolve(pin)}^{{tree}}").decode().strip() + + def changed_paths(self, old: str, new: str) -> list[str]: + data = self._git("diff", "--name-only", "-z", "--no-renames", self.resolve(old), self.resolve(new)) + return sorted({path.decode("utf-8") for path in data.split(b"\0") if path}) + + def parents(self, pin: str) -> tuple[str, ...]: + return tuple(self._git("rev-list", "--parents", "-n", "1", self.resolve(pin)).decode().split()[1:]) + + def files(self, pin: str) -> dict[str, dict[str, str]]: + pin = self.resolve(pin) + if pin in self._inventories: + return {path: dict(meta) for path, meta in self._inventories[pin].items()} + result = {} + for row in self._git("ls-tree", "-r", "-z", "--full-tree", pin).split(b"\0"): + if not row: + continue + meta, name = row.split(b"\t", 1) + mode, kind, oid = meta.decode("ascii").split() + path = name.decode("utf-8") + if kind == "blob" and mode in ("100644", "100755"): + result[path] = {"mode": mode, "oid": oid} + self._inventories[pin] = result + return {path: dict(meta) for path, meta in result.items()} + + def read(self, pin: str, path: str) -> bytes: + pure = PurePosixPath(path) + if not path or pure.is_absolute() or ".." in pure.parts or "\\" in path or "\0" in path: + raise SourceError("source path escapes the fixed tree") + files = self.files(pin) + if path not in files: + raise SourceError("source path is not a regular committed file") + return self._git("cat-file", "blob", files[path]["oid"]) + + def export(self, pin: str, destination: str | Path) -> Path: + root = Path(destination).resolve() + root.mkdir(parents=True, exist_ok=True) + if any(root.iterdir()): + raise SourceError("source export destination must be empty") + inventory = self.files(pin) + # Archive export attributes can remove or rewrite committed blobs. + # Read the actual tree objects in one batch instead; export and read + # therefore have identical evidence bytes and inventory membership. + proc = subprocess.run(["git", "-C", str(self.path), "cat-file", "--batch"], + input="".join(f"{meta['oid']}\n" for meta in inventory.values()).encode(), + capture_output=True, check=False) + if proc.returncode: + raise SourceError(f"Git source export failed (exit {proc.returncode})") + offset = 0 + for relative, meta in inventory.items(): + end = proc.stdout.find(b"\n", offset) + header = proc.stdout[offset:end].split() if end >= offset else [] + if len(header) != 3 or header[0].decode("ascii") != meta["oid"] or header[1] != b"blob" or not header[2].isdigit(): + raise SourceError("committed source export object is invalid") + size = int(header[2]) + content = proc.stdout[end + 1:end + 1 + size] + offset = end + 1 + size + if len(content) != size or proc.stdout[offset:offset + 1] != b"\n": + raise SourceError("committed source export is incomplete") + offset += 1 + target = (root / relative).resolve() + if not target.is_relative_to(root): + raise SourceError("committed source path escapes destination") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + target.chmod(0o755 if meta["mode"] == "100755" else 0o644) + if offset != len(proc.stdout): + raise SourceError("committed source export contains unexpected objects") + return root + + +class ForgeProvider: + """Optional review transport. Source identity always comes from GitSource.""" + + kind = "none" + + def __init__(self, project: str = "", *, host: str = "", token: str = "", + request: Callable | None = None): + self.project = project + self.host = host.rstrip("/") + self._token = token # runtime only; never in public spec/receipts + self._request = request or self._http + + def _http(self, method: str, url: str, data: dict | None = None) -> dict: + headers = {"Accept": "application/json"} + if self._token: + headers.update(self.auth_headers()) + body = None if data is None else json.dumps(data).encode() + if body is not None: + headers["Content-Type"] = "application/json" + try: + with urllib.request.urlopen(urllib.request.Request(url, body, headers, method=method), timeout=60) as reply: + return json.loads(reply.read()) + except Exception as exc: + raise SourceError(f"{self.kind} review transport failed") from exc + + def auth_headers(self) -> dict: + return {} + + def open_review(self, *, title: str, body: str, head: str, base: str) -> dict: + raise SourceError("this repository has no configured review forge; use explicit local acceptance") + + def review(self, number: int) -> dict: + raise SourceError("this repository has no configured review forge") + + def acceptance(self, number: int, expected_head: str) -> dict: + """Merged identity only; CI is separately checked, never inferred.""" + if not COMMIT_ID.fullmatch(expected_head): + raise SourceError("expected review head must be a full commit") + row = self.review(number) + if self.kind == "github": + merged = row.get("merged") is True + head = (row.get("head") or {}).get("sha") + elif self.kind == "gitlab": + merged = row.get("state") == "merged" + head = row.get("sha") + else: + raise SourceError("no forge acceptance is available; use local acceptance") + publication = row.get("merge_commit_sha") or row.get("squash_commit_sha") + if not merged or head != expected_head or not isinstance(publication, str) or not COMMIT_ID.fullmatch(publication): + raise SourceError("forge review is not merged at the frozen head/publication commit") + return {"schema_version": 1, "kind": "forge_review_acceptance", "forge": self.kind, + "project": self.project, "number": int(number), "review_head": expected_head, + "publication_head": publication, "merged": True, + "ci_verified": False, "claim": "Merged review identity; independent source/native/CI gates remain separate."} + + +class GitHubForge(ForgeProvider): + kind = "github" + + def __init__(self, project: str, **kwargs): + super().__init__(project, host=kwargs.pop("host", "https://api.github.com"), **kwargs) + if len(project.split("/")) != 2: + raise SourceError("GitHub project must be owner/repository") + + def auth_headers(self) -> dict: + return {"Authorization": f"Bearer {self._token}"} + + def open_review(self, *, title: str, body: str, head: str, base: str) -> dict: + return self._request("POST", f"{self.host}/repos/{self.project}/pulls", + {"title": title, "body": body, "head": head, "base": base}) + + def review(self, number: int) -> dict: + return self._request("GET", f"{self.host}/repos/{self.project}/pulls/{int(number)}") + + +class GitLabForge(ForgeProvider): + kind = "gitlab" + + def auth_headers(self) -> dict: + return {"PRIVATE-TOKEN": self._token} + + @property + def project_url(self) -> str: + if not self.host or not self.project or any(p in ("", ".", "..") for p in self.project.split("/")): + raise SourceError("GitLab project/host is invalid") + return f"{self.host}/api/v4/projects/{urllib.parse.quote(self.project, safe='')}" + + def open_review(self, *, title: str, body: str, head: str, base: str) -> dict: + return self._request("POST", f"{self.project_url}/merge_requests", + {"title": title, "description": body, "source_branch": head, "target_branch": base}) + + def review(self, number: int) -> dict: + return self._request("GET", f"{self.project_url}/merge_requests/{int(number)}") + + +def forge_provider(kind: str, project: str = "", **kwargs) -> ForgeProvider: + providers = {"none": ForgeProvider, "github": GitHubForge, "gitlab": GitLabForge} + try: + return providers[kind](project, **kwargs) + except KeyError as exc: + raise SourceError("unsupported review forge") from exc diff --git a/src/infermatrix_copilot/kb_service/init_feature_discovery.py b/src/infermatrix_copilot/kb_service/init_feature_discovery.py index b64748225..44ce53667 100644 --- a/src/infermatrix_copilot/kb_service/init_feature_discovery.py +++ b/src/infermatrix_copilot/kb_service/init_feature_discovery.py @@ -27,12 +27,14 @@ from .init_support import InitError, InitRecord, generate, inputs_digest from .models import ModelUnavailable -VERSION = "feature-discovery-v1" +VERSION = "feature-discovery-v2" MAX_PACKET_CHARS = 24000 MAX_CONFIGURED_PACKET_CHARS = 192000 DEFAULT_START_INTERVAL_S = 15.0 MAX_CANDIDATES = 24 MAX_ATTEMPTS = 4 +MAX_RESIDUAL_ROUNDS = 2 +MAX_SATURATION_SPLITS = 2 MAX_BASELINE_CHARS = 16000 CATALOG_BOUNDARY_VERSION = "catalog-boundary-v1" COMPACT_REPORT_VERSION = "feature-discovery-compact-v1" @@ -73,6 +75,9 @@ def discovery_run_config(rt, environ=None): "zcode_reasoning_level": level} SYSTEM_DISCOVER = """Discover distinct software capabilities from the supplied pinned evidence. First documentation establishes a baseline; source/test packets expand it. +Use the entry_contract_leads to explore actual behavior and contracts, not +only the document baseline's words. In a residual round focus on unexplained +entries; a lead is not itself a feature, and its relationships may be unknown. Read actual bodies, not just headings, filenames or scan lists. A capability has meaningful observable behavior or an internal contract/lifecycle. A file, function, UI page or setting is not automatically a separate capability. Link @@ -165,7 +170,8 @@ def write_full_discovery_report(state_dir, report): def compact_discovery_report(report, artifact): keys = ("schema_version", "repo", "pin", "run_config", "complete", "done", "catalog_sha256", "catalog_renderer_version", "feature_ids", "features", "owner_requests", "counts", "statistics", "scan_summary", - "index_sha256", "limits", "historical_denominator", "facet_denominator", "pacing") + "index_sha256", "limits", "historical_denominator", "facet_denominator", "pacing", + "unit_summary", "residual_summary", "evidence_bundles", "scan_mode", "scan_paths") compact = {key: deepcopy(report[key]) for key in keys if key in report} audit = report["catalog_boundary_audit"] compact["catalog_boundary_audit"] = {key: deepcopy(audit[key]) for key in ("identity", "done", "counts") if key in audit} @@ -472,7 +478,7 @@ class DiscoveryEngine: results and use a shared generator/judge concurrency ceiling. """ def __init__(self, index, *, seeds, owners, state, call, save, concurrency=13, repository="", pin="", - packet_chars=None): + packet_chars=None, scan_paths=None): packet_chars = MAX_PACKET_CHARS if packet_chars is None else packet_chars if isinstance(packet_chars, bool) or not isinstance(packet_chars, int) \ or not MAX_PACKET_CHARS <= packet_chars <= MAX_CONFIGURED_PACKET_CHARS: @@ -481,10 +487,21 @@ def __init__(self, index, *, seeds, owners, state, call, save, concurrency=13, r self.call, self.save, self.concurrency = call, save, concurrency self.repository, self.pin = repository, pin self.packet_chars = packet_chars + self.scan_paths = None if scan_paths is None else frozenset(scan_paths) + if self.scan_paths is not None and any(path not in index.entries for path in self.scan_paths): + raise ValueError("incremental discovery paths are outside the frozen index") self.state.setdefault("tasks", {}) self.state.setdefault("candidates", {}) self.state.setdefault("reviews", {}) self.state.setdefault("failures", []) + # Attempts survive changed evidence and supplemental rounds. Native + # judgments remain bound to the candidate hash they actually assessed. + self.state.setdefault("review_attempts", {}) + for key, checked in self.state["reviews"].items(): + self.state["review_attempts"][key] = max(self.state["review_attempts"].get(key, 0), checked.get("attempts", 0)) + from .feature_discovery_index import contract_units + self.units = [unit for unit in contract_units(index) + if self.scan_paths is None or unit["path"] in self.scan_paths] def _baseline(self): return [{"catalog_status": "existing_feature", **{k: row.get(k) for k in ("id", "title", "owner", "source_globs")}} for row in self.seeds] + [ @@ -492,6 +509,8 @@ def _baseline(self): def _packets(self, kind): chunks = [c for c in self.index.chunks if (c["kind"] == "doc") == (kind == "doc")] + if self.scan_paths is not None: + chunks = [c for c in chunks if c["path"] in self.scan_paths] # Unassociated implementation regions first; every declared chunk still runs. from .knowledge_coverage import matches if kind != "doc": @@ -515,8 +534,15 @@ def _extract(self, key, packet, baseline): payload = {"repository": self.repository, "pin": self.pin, "round": key.split(":")[0], "baseline": _bounded_baseline(baseline, "\n".join(c["text"] for c in evidence)), "owners": self.owners, "files": evidence} + leads = [unit for unit in self.units if any( + c["path"] == unit["path"] and c.get("start", 0) <= unit["start"] <= c.get("end", 0) + for c in evidence)] + payload["entry_contract_leads"] = leads[:128] + payload["entry_contract_lead_count"] = len(leads) + payload["lead_limitations"] = "Localization leads, not certified features or runtime/test relationships." last_error = self.state["tasks"].get(key, {}).get("reason", "") - for attempt in range(MAX_ATTEMPTS): + attempts = self.state["tasks"].get(key, {}).get("attempts", 0) + for attempt in range(attempts, MAX_ATTEMPTS): try: reply = self.call("generator", SYSTEM_DISCOVER, _prompt({**payload, "repair_reason": last_error}), validate_candidates) proof = receipt(reply) @@ -533,7 +559,7 @@ def _extract(self, key, packet, baseline): for ref in row["evidence"]): row["invalid_reason"] = "citation not present in offered pinned lines" row["generator_receipts"] = [proof] - row["origin_rounds"] = [payload["round"]] + row["origin_rounds"] = ["source", payload["round"]] if payload["round"].startswith("residual") else [payload["round"]] checked.append(row) return {"status": "complete", "attempts": attempt + 1, "candidates": checked, "invalid_candidates": invalid, "receipt": proof, "candidate_limit_reached": len(reply.data["candidates"]) == MAX_CANDIDATES} @@ -543,7 +569,8 @@ def _extract(self, key, packet, baseline): return {"status": "unknown", "attempts": attempt + 1, "reason": str(exc), "candidates": []} except ValueError as exc: last_error = str(exc) - return {"status": "unknown", "attempts": MAX_ATTEMPTS, "reason": last_error, "candidates": []} + return {"status": "unknown", "attempts": MAX_ATTEMPTS, + "reason": last_error or "initial extraction and three corrections exhausted", "candidates": []} def _collect(self, task, *, invalidate=True): for incoming in task.get("candidates", []): @@ -553,6 +580,12 @@ def _collect(self, task, *, invalidate=True): if old is None: self.state["candidates"][key] = row continue + # Rediscovery with exactly the same claim/evidence is not a new + # candidate revision. Retain the original native approval binding; + # the extra extraction is still preserved in its task archive. + if all(old.get(field) == row.get(field) for field in + ("title", "description", "owner", "relation", "related_id", "aliases", "evidence", "invalid_reason")): + continue # Same stable ID may gather implementation evidence from another layer. names = {unicodedata.normalize("NFKC", name).casefold().strip() for name in [old["title"], *old.get("aliases", [])]} @@ -569,52 +602,97 @@ def _collect(self, task, *, invalidate=True): if old.get("invalid_reason") and not row.get("invalid_reason"): old.pop("invalid_reason", None) if invalidate and _hash(old) != previous_hash: - self.state["reviews"].pop(key, None) + checked = self.state["reviews"].get(key, {}) + self.state["review_attempts"][key] = max(self.state["review_attempts"].get(key, 0), checked.get("attempts", 0)) + + @staticmethod + def _packet_key(kind, packet): + return kind + ":" + _hash([{k: c.get(k) for k in + ("path", "start", "end", "sha256", "status", "character_start", "character_end")} for c in packet]) + + @staticmethod + def _split_packet(packet): + """Deterministic smaller evidence, never another sample of the same cap.""" + if len(packet) > 1: + middle = len(packet) // 2 + return [packet[:middle], packet[middle:]] + item = packet[0] + lines = item.get("text", "").split("\n") + if len(lines) < 2 or item.get("partial_line") or item.get("start", 0) < 1: + return [] + middle = len(lines) // 2 + return [[{**item, "text": "\n".join(lines[:middle]), "end": item["start"] + middle - 1}], + [{**item, "text": "\n".join(lines[middle:]), "start": item["start"] + middle}]] + + def _rebuild_candidates(self): + self.state["candidates"] = {} + for _, task in sorted(self.state["tasks"].items()): + self._collect(task, invalidate=False) + for key, override in self.state.get("repairs", {}).items(): + aggregated = self.state["candidates"].get(key) + if aggregated and _hash(aggregated) == override["base_sha256"]: + self.state["candidates"][key] = deepcopy(override["candidate"]) + for key, checked in list(self.state["reviews"].items()): + current = self.state["candidates"].get(key) + if current and checked.get("candidate_sha256") and checked["candidate_sha256"] != _hash(current): + self.state["review_attempts"][key] = max(self.state["review_attempts"].get(key, 0), checked.get("attempts", 0)) + + def _run_packet_work(self, kind, packets): + ready, visited = deque(), set() + + def schedule(packet, depth=0, parent=""): + key = self._packet_key(kind, packet) + if key in visited: + return + visited.add(key) + task = self.state["tasks"].get(key, {}) + if task.get("status") in ("complete", "unknown"): + if task.get("candidate_limit_reached"): + expand(key, packet, depth) + return + ready.append((key, packet, depth, parent)) + + def expand(key, packet, depth): + task = self.state["tasks"][key] + children = self._split_packet(packet) if depth < MAX_SATURATION_SPLITS else [] + task["split_depth"] = depth + task["split_children"] = [self._packet_key(kind, child) for child in children] + task["overflow_unknown"] = not bool(children) + for child in children: + schedule(child, depth + 1, key) + + for packet in packets: + schedule(packet) + baseline, stopped = self._baseline(), False + with ThreadPoolExecutor(max_workers=self.concurrency) as pool: + pending = {} + while ready or pending: + while not stopped and ready and len(pending) < self.concurrency: + key, packet, depth, parent = ready.popleft() + pending[pool.submit(self._extract, key, packet, baseline)] = (key, packet, depth, parent) + if not pending: + break + future = next(as_completed(pending)) + key, packet, depth, parent = pending.pop(future) + try: + task = future.result() + except BudgetExhausted: + stopped = True + self.state["tasks"].setdefault(key, {}).update(status="pending", reason="budget exhausted") + else: + task.update(split_depth=depth, split_parent=parent) + self.state["tasks"][key] = task + self._collect(task) + if task.get("candidate_limit_reached"): + expand(key, packet, depth) + self.save() + self._rebuild_candidates() + self.save() + return not stopped def scan(self): for kind in ("doc", "source"): - work = [] - for packet in self._packets(kind): - key = kind + ":" + _hash([{k: c.get(k) for k in ("path", "start", "end", "sha256", "status", "character_start", "character_end")} for c in packet]) - if self.state["tasks"].get(key, {}).get("status") in ("complete", "unknown"): - continue - work.append((key, packet)) - baseline = self._baseline() - stopped = False - with ThreadPoolExecutor(max_workers=self.concurrency) as pool: - # Bound submitted work as well as executing calls. Budget exhaustion - # stops further scheduling while already-running calls checkpoint. - pending, cursor = {}, 0 - while cursor < len(work) or pending: - while not stopped and cursor < len(work) and len(pending) < self.concurrency: - key, packet = work[cursor]; cursor += 1 - pending[pool.submit(self._extract, key, packet, baseline)] = key - if not pending: - break - future = next(as_completed(pending)); key = pending.pop(future) - try: - task = future.result() - except BudgetExhausted: - stopped = True - self.state["tasks"][key] = {"status": "pending", "reason": "budget exhausted"} - else: - self.state["tasks"][key] = task - self._collect(task) - self.save() - # Stable aggregation independent of concurrent completion order. - self.state["candidates"] = {} - for _, task in sorted(self.state["tasks"].items()): - self._collect(task, invalidate=False) - for key, override in self.state.get("repairs", {}).items(): - aggregated = self.state["candidates"].get(key) - if aggregated and _hash(aggregated) == override["base_sha256"]: - self.state["candidates"][key] = deepcopy(override["candidate"]) - for key, checked in list(self.state["reviews"].items()): - current = self.state["candidates"].get(key) - if checked.get("supported") == "yes" and current and checked.get("candidate_sha256") != _hash(current): - self.state["reviews"].pop(key) - self.save() - if stopped: + if not self._run_packet_work(kind, self._packets(kind)): self.state["scan_complete"] = False return False self.state["scan_complete"] = True @@ -669,7 +747,11 @@ def _review(self, rows, baseline, *, audit=None): return results def review(self): - pending = [row for key, row in sorted(self.state["candidates"].items()) if key not in self.state["reviews"]] + pending = [row for key, row in sorted(self.state["candidates"].items()) + if (key not in self.state["reviews"] or + self.state["reviews"][key].get("candidate_sha256") and + self.state["reviews"][key]["candidate_sha256"] != _hash(row)) + and self.state["review_attempts"].get(key, 0) < MAX_ATTEMPTS] baseline = self._baseline() batches = [pending[n:n + 12] for n in range(0, len(pending), 12)] stopped = False @@ -687,7 +769,8 @@ def review(self): decisions = {r["id"]: {"supported": "unsure", "relation": "unknown", "related_id": "", "reason": str(exc), "repair_blocked": True} for r in rows} for row in rows: result = decisions[row["id"]] - result["attempts"] = 1 + result["attempts"] = self.state["review_attempts"].get(row["id"], 0) + 1 + self.state["review_attempts"][row["id"]] = result["attempts"] self.state["reviews"][row["id"]] = result self.save() if stopped: @@ -721,7 +804,7 @@ def _repair_parallel(self, baseline): # share the same ceiling, with every draft saved before its judge starts. def eligible(key): row, decision = self.state["candidates"][key], self.state["reviews"].get(key, {}) - return decision.get("supported") != "yes" and decision.get("attempts", 0) < MAX_ATTEMPTS \ + return decision.get("supported") != "yes" and self.state["review_attempts"].get(key, decision.get("attempts", 0)) < MAX_ATTEMPTS \ and not decision.get("repair_blocked") \ and any(ref["path"] in self.index.production for ref in row["evidence"]) @@ -734,7 +817,7 @@ def eligible(key): key = ready.popleft() original = deepcopy(self.state["candidates"][key]) decision = deepcopy(self.state["reviews"].get(key, {})) - attempt = decision.get("attempts", 0) + 1 + attempt = self.state["review_attempts"].get(key, decision.get("attempts", 0)) + 1 repair_base = self.state.get("repairs", {}).get(key, {}).get("base_sha256", _hash(original)) draft = self.state.get("repair_drafts", {}).get(key) if draft and draft["base_sha256"] == _hash(original) and draft["attempt"] == attempt: @@ -774,11 +857,137 @@ def eligible(key): decision = {"supported": "unsure", "relation": "unknown", "related_id": "", "reason": str(exc), "attempts": attempt} self.state.get("repair_drafts", {}).pop(key, None) self.state["reviews"][key] = decision + self.state["review_attempts"][key] = max(self.state["review_attempts"].get(key, 0), decision.get("attempts", 0)) self.save() if eligible(key): ready.append(key) return not stopped + def unit_resolution(self, features=None, outcomes=None): + """Only independently accepted line anchors explain an inventory unit. + + File/glob association, an unknown candidate and a neighboring symbol + cannot hide another unexplained entry in the same production file. + """ + if features is None or outcomes is None: + features, outcomes = self.catalog("repos/discovery") + formal = {f["id"] for f in features} + mapping = {unit["id"]: [] for unit in self.units} + accepted = {r["id"]: r for r in outcomes if r["status"] == "accepted"} + by_path = {} + for unit in self.units: + if unit["status"] == "ready": + by_path.setdefault(unit["path"], []).append(unit) + + def target(row): + key, visited = row["id"], set() + while key not in formal and key not in visited: + visited.add(key) + related = accepted.get(key, {}).get("related_id", "") + if not related: + return "" + key = related + return key if key in formal else "" + + for row in accepted.values(): + feature_id = target(row) + if not feature_id: + continue + statement = " ".join(str(row.get(key, "")) for key in ("title", "description")) + for ref in row["evidence"]: + cited = [unit for unit in by_path.get(ref["path"], []) + if ref["start"] <= unit["start"] <= ref["end"]] + anchors = {unit["start"] for unit in cited} + for unit in cited: + name = unit["name"].rsplit(".", 1)[-1] + named = unit["kind"] == "public_contract" and re.search( + r"(?") _OWNER = re.compile(r"[a-z0-9][a-z0-9-]{0,40}") @@ -177,6 +177,27 @@ def _restore_progress(self, previous: InitRecord | None) -> list[str]: def _publish(self, changed: dict[str, str]) -> InitRecord: targets = self.record.coverage.get("knowledge", {}).get("targets", {}) + if self.STAGE == "knowledge" and getattr(self.rt, "portable_spec", None): + from .knowledge_coverage import coverage_targets_met + + owners = self.record.coverage.get("knowledge", {}).get("owners", {}) + self.record.coverage["foundation"] = { + "mode": "partial", "tier": "foundation", "init_complete": False, + "foundation_targets_met": targets.get("met") is True, + "structural_targets_met": coverage_targets_met(targets, "partial"), + "catalog_binding": self.record.discovery.get("catalog_binding", {}), + "unknown_features": {key: list(item.get("missing_facets", [])) + for key, item in targets.get("features", {}).get("items", {}).items() + if item.get("missing_facets")}, + "unknown_owners": {key: [facet for facet, status in item.get("facets", {}).items() + if status != "covered"] for key, item in owners.items() + if any(status != "covered" for status in item.get("facets", {}).values())}, + "unknown_core_files": list(targets.get("core", {}).get("missing_files", [])), + "claim": "Accepted foundation retains supported context and explicit unknowns; " + "it does not establish seven-facet final acceptance or executed tests."} + self.record.notes.append("portable foundation preview: unsupported facets remain unknown; " + "final semantic-depth and structural targets are unchanged") + return super()._publish(changed) if self.STAGE == "knowledge" and self.foundation_mode == "partial": from .foundation_publication import freeze_receipt freeze_receipt(self, changed) @@ -368,6 +389,17 @@ def _build(self, tree: Path) -> InitRecord: source = preferred_sources(self, tree, feature.id, source_paths, MAX_SOURCE_BYTES - test_bytes) source.extend(tests) docs = self._sources(tree, list(feature.docs), MAX_DOC_BYTES) + discovery = self._discovery_catalog() + bundle = discovery.get("evidence_bundles", {}).get(feature.id) + if bundle: + from .evidence_bundle import materialize_bundle, bounded_spans + try: + spans = materialize_bundle(bundle, tree, pin=self.record.pin, + catalog_hash=discovery["catalog_sha256"]) + except (ValueError, OSError, UnicodeError) as exc: + return self._blocked([f"discovery evidence handoff: {exc}"]) + source = bounded_spans([s for s in spans if s["kind"] != "doc"] + source, MAX_SOURCE_BYTES) + docs = bounded_spans([s for s in spans if s["kind"] == "doc"] + docs, MAX_DOC_BYTES) if not source: self.record.unfinished.append(f"feature {feature.id}: missing readable source") continue @@ -582,8 +614,8 @@ def _render_section(self, section, entries): text = text.replace("\n\n", "\n\nInference / 设计推断（非作者历史意图）：\n\n", 1) citations = [] for entry in entries: - url = (f"https://github.com/{self.lifecycle.full_name}/blob/{self.record.pin}/" - f"{quote(entry.path, safe='/')}#L{entry.start}-L{entry.end}") + from .source_links import source_link + url = source_link(self.lifecycle.full_name, self.record.pin, entry.path, entry.start, entry.end) citations.append(f"[{entry.path}:L{entry.start}–L{entry.end}]({url})") text += "\n\nSources / 来源：" + ", ".join(citations) + "\n" return text diff --git a/src/infermatrix_copilot/kb_service/init_knowledge_depth.py b/src/infermatrix_copilot/kb_service/init_knowledge_depth.py index e4fe3065e..f14e9f34d 100644 --- a/src/infermatrix_copilot/kb_service/init_knowledge_depth.py +++ b/src/infermatrix_copilot/kb_service/init_knowledge_depth.py @@ -72,10 +72,19 @@ def _init_identity(self) -> str: def _input_options(self) -> dict: foundation = {} if self.foundation_mode == "partial": - from .foundation_publication import foundation_handoff - binding = foundation_handoff(self.foundation_record_path, knowledge=self.rt.knowledge, - baseline=self._base_sha, repo=self.lifecycle.repo, pin=self._knowledge_run_pin, - publisher=self._publisher()) + if getattr(self.rt, "portable_spec", None): + from .portable_init import check_local_acceptance + record = InitRecord.load(self.rt.state_dir, self.lifecycle.repo, "knowledge") + if record is None or record.status != "accepted" or record.pin != self._knowledge_run_pin: + raise InitError("portable depth requires an accepted foundation at the same source pin") + head = check_local_acceptance(self.rt, record) + binding = {"local_head_sha": head, "inputs_digest": record.inputs_digest, + "catalog_binding": record.discovery.get("catalog_binding"), "init_complete": False} + else: + from .foundation_publication import foundation_handoff + binding = foundation_handoff(self.foundation_record_path, knowledge=self.rt.knowledge, + baseline=self._base_sha, repo=self.lifecycle.repo, pin=self._knowledge_run_pin, + publisher=self._publisher()) self._foundation_binding = binding foundation = {"foundation_mode": "partial", "foundation": binding} if self.acceptance_mode == "lightweight": @@ -102,7 +111,7 @@ def _base_for_run(self, latest: str) -> str: if previous and previous.depth: if self.retry_unfinished and previous.status == "published": raise InitError("published depth batch is immutable; use a new state directory after merging its PR") - if not re.fullmatch(r"[0-9a-f]{40}", previous.kb_base_sha): + if not re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", previous.kb_base_sha): raise InitError("depth checkpoint has an invalid knowledge baseline") if self.rt.knowledge._git("cat-file", "-t", previous.kb_base_sha).decode().strip() != "commit": raise InitError("depth checkpoint knowledge baseline is unavailable") @@ -186,7 +195,7 @@ def _build(self, tree: Path) -> InitRecord: policy_text = self.overlay.get(policy_path) or self.rt.knowledge.show(self._base_sha, policy_path) context = DepthContext(tree, inventory(tree, policy), mode="lightweight", cache_path=self.depth_index_path, pin=self.record.pin, - policy_sha256=digest(policy_text)) + policy_sha256=digest(policy_text), discovery=self._discovery_catalog()) if getattr(self, "_repair_guidance", None): from .depth_inputs import load_repair_guidance checked_guidance = load_repair_guidance( @@ -195,7 +204,7 @@ def _build(self, tree: Path) -> InitRecord: if checked_guidance["guidance_sha256"] != self._repair_guidance["guidance_sha256"]: raise InitError("repair guidance changed after checkpoint identity was captured") else: - context = DepthContext(tree, inventory(tree, policy)) + context = DepthContext(tree, inventory(tree, policy), discovery=self._discovery_catalog()) by_id = {f.id: f for f in policy.features} selected_features = tuple(by_id[fid] for fid in dict.fromkeys(self.feature_ids)) if self.feature_ids else policy.features # Finish a fair first pass before targeted repairs. Subscription mode diff --git a/src/infermatrix_copilot/kb_service/init_stages.py b/src/infermatrix_copilot/kb_service/init_stages.py index 39a7e31e8..73d0c0c13 100644 --- a/src/infermatrix_copilot/kb_service/init_stages.py +++ b/src/infermatrix_copilot/kb_service/init_stages.py @@ -167,8 +167,8 @@ def run_stage(rt: InitRuntime, lifecycle, stage: str, *, dry_run: bool, pin: str foundation_mode == "strict" and foundation_record_path is not None): raise InitError("foundation_mode must be strict or partial; a foundation record requires partial mode") if foundation_mode == "partial" and ( - stage not in ("knowledge", "knowledge-deepen") or not unlimited_subscription - or bool(foundation_record_path) != (stage == "knowledge-deepen")): + stage not in ("knowledge", "knowledge-deepen") or (not unlimited_subscription and not getattr(rt, "portable_spec", None)) + or (bool(foundation_record_path) != (stage == "knowledge-deepen") and not getattr(rt, "portable_spec", None))): raise InitError("partial foundation requires unlimited knowledge, or knowledge-deepen with its published foundation record") if feature_ids and (stage != "knowledge-deepen" or not isinstance(feature_ids, tuple) or any(not isinstance(f, str) or not f for f in feature_ids)): @@ -500,6 +500,8 @@ def run(self) -> InitRecord: upstream = rt.upstream(lc.repo, lc.full_name) upstream.sync() pin = upstream.resolve(self.pin or chain.pin or "HEAD") + if getattr(rt, "portable_spec", None) and pin != rt.portable_spec.source_pin: + raise InitError("portable source pin differs from the accepted batch; create a new batch") self._discovery_gate(chain, pin) self.overlay = dict(chain.repo_files) input_options = dict(self._input_options()) @@ -645,9 +647,20 @@ def _chain(self) -> _Chain: chain.knowledge.clear() # main holds it, and every stage before it chain.repo_files.clear() parts.append(f"{stage}:merged:{number}") + elif record.status == "accepted" and getattr(self.rt, "portable_spec", None): + from .portable_init import check_local_acceptance + try: + head = check_local_acceptance(self.rt, record) + except InitError as exc: + chain.problems.append(str(exc)) + continue + parts.append(f"{stage}:local-accepted:{head}") elif record.status == "empty": parts.append(f"{stage}:empty:{record.inputs_digest}") elif record.status == "dry_run": + if getattr(self.rt, "portable_spec", None): + chain.problems.append(f"accept the {stage} preview before starting {self.STAGE}") + continue if not self.dry_run: chain.problems.append(f"the {stage} stage was only a dry run; publish it and merge its PR " f"before publishing {self.STAGE}") @@ -814,7 +827,17 @@ def _discovery_gate(self, chain: _Chain, pin: str) -> None: if state != "MERGED": chain.problems.append("merge the feature-discovery PR before continuing") return + elif record.status == "accepted" and getattr(self.rt, "portable_spec", None): + from .portable_init import check_local_acceptance + try: + check_local_acceptance(self.rt, record) + except InitError as exc: + chain.problems.append(str(exc)) + return elif record.status == "dry_run" and self.dry_run: + if getattr(self.rt, "portable_spec", None): + chain.problems.append("accept the feature-discovery catalog before continuing") + return try: files = _dry_run_files(record) except InitError as exc: @@ -1360,6 +1383,9 @@ def _publish(self, changed: dict[str, str]) -> InitRecord: dest = InitRecord.path(rt.state_dir, lc.repo, stage).with_name(f"{stage}-dryrun") InitPublisher.dry_run(dest, changed, title=title, body=body) record.pr = {"dry_run_dir": str(dest)} + if getattr(rt, "portable_spec", None): + record.pr["checked_files_sha256"] = {p: hashlib.sha256(t.encode()).hexdigest() + for p, t in sorted(changed.items())} record.status = "dry_run" else: prepared = save_prepared( diff --git a/src/infermatrix_copilot/kb_service/init_support.py b/src/infermatrix_copilot/kb_service/init_support.py index 4213356bf..cbdc2590c 100644 --- a/src/infermatrix_copilot/kb_service/init_support.py +++ b/src/infermatrix_copilot/kb_service/init_support.py @@ -49,7 +49,7 @@ AUTHOR_ENV = "KB_INIT_GIT_AUTHOR" CLONE_ENV = "KB_INIT_KNOWLEDGE_CLONE" _AUTHOR = re.compile(r"(?P[^<>]+?)\s*<(?P[^<>\s]+@[^<>\s]+)>") -_SHA = re.compile(r"[0-9a-f]{40}") +_SHA = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") # what an init PR may write, repository-relative INIT_PATHS = ( re.compile(r"knowledge/(?:repos/[A-Za-z0-9._-]+|general)/(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+\.(?:md|yaml)"), @@ -169,8 +169,12 @@ def sync(self) -> None: listing = self._git("ls-remote", "--symref", "origin", "HEAD").decode(errors="replace") match = re.search(r"^ref:\s+(refs/heads/\S+)\s+HEAD$", listing, re.MULTILINE) if match is None: - raise InitError(f"{self.full_name} does not report a default branch") - self._git("symbolic-ref", "HEAD", match.group(1)) + # A local source can intentionally have a detached HEAD and no origin. + # Fetch that exact object; do not fabricate a default branch. + self._git("fetch", "--quiet", "origin", "+HEAD:refs/kb/source-head") + self._git("update-ref", "--no-deref", "HEAD", self._git("rev-parse", "refs/kb/source-head").decode().strip()) + else: + self._git("symbolic-ref", "HEAD", match.group(1)) def resolve(self, ref: str) -> str: sha = self._git("rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}").decode().strip() @@ -179,24 +183,13 @@ def resolve(self, ref: str) -> str: return sha def export(self, pin: str, dest: Path) -> Path: - """The tree at ``pin`` as plain files (``git archive``; members that - would land outside ``dest`` are refused).""" - data = self._git("archive", "--format=tar", pin) - dest = Path(dest) - dest.mkdir(parents=True, exist_ok=True) - root = dest.resolve() - with tarfile.open(fileobj=io.BytesIO(data)) as archive: - for member in archive.getmembers(): - target = (dest / member.name).resolve() - if not target.is_relative_to(root) or member.issym() or member.islnk(): - continue # links and escapes are not docs or code we read - if member.isdir(): - target.mkdir(parents=True, exist_ok=True) - elif member.isfile(): - target.parent.mkdir(parents=True, exist_ok=True) - source = archive.extractfile(member) - target.write_bytes(source.read() if source is not None else b"") - return dest + """Materialize exact committed blobs, without Git archive attributes.""" + from .git_source import GitSource + from .sources import SourceError + try: + return GitSource(self.path).export(pin, dest) + except SourceError as exc: + raise InitError(str(exc)) from exc def observer(self, pin: str, *, pull=None) -> PinnedObserver: return PinnedObserver(self.path, self.full_name, pin, pull=pull) @@ -254,7 +247,7 @@ def collect_docs(root: Path, globs: tuple[str, ...], *, max_files: int = MAX_DOC @dataclass class InitRuntime: - settings: Any + settings: Any = field(repr=False) state_dir: Path registry: dict gateway: ModelGateway @@ -263,7 +256,7 @@ class InitRuntime: knowledge: Any # sources.KnowledgeRepo github: Any # sources.GitHubReader prices: dict[str, Price] = field(default_factory=load_prices) - environ: Mapping[str, str] = field(default_factory=lambda: dict(os.environ)) + environ: Mapping[str, str] = field(default_factory=lambda: dict(os.environ), repr=False) clock: Callable[[], float] = time.time gh_run: Callable[..., subprocess.CompletedProcess] = subprocess.run upstream_remote: Callable[[str], str] | None = None # full_name -> clone URL (tests) @@ -271,6 +264,8 @@ class InitRuntime: subscription_generator: bool = False unlimited_subscription: bool = False discovery_concurrency: int = 13 + portable_spec: Any = None + discovery_paths: tuple[str, ...] | None = None @property def init_dir(self) -> Path: diff --git a/src/infermatrix_copilot/kb_service/knowledge_coverage.py b/src/infermatrix_copilot/kb_service/knowledge_coverage.py index 984084763..f63efd111 100644 --- a/src/infermatrix_copilot/kb_service/knowledge_coverage.py +++ b/src/infermatrix_copilot/kb_service/knowledge_coverage.py @@ -51,8 +51,8 @@ def feature_metadata(text: str, feature) -> str: return page.render() -_FILE = re.compile(r"\n(.*?)\n", re.S) -_FACET = re.compile(r"\n(.*?)\n", re.S) +_FACET = re.compile(r"") _SLUG = re.compile(r"[a-z0-9][a-z0-9-]{0,39}") _LEXICAL_NO_CODE = re.compile(r'''"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`|/\*[\s\S]*?\*/|//[^\n]*|''') @@ -167,7 +167,9 @@ def load_policy(text: str, repo_dir: str) -> CoveragePolicy: def matches(path: str, patterns: tuple[str, ...]) -> bool: - return any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns) + return any(fnmatch.fnmatchcase(path, pattern) + or (pattern.startswith("**/") and fnmatch.fnmatchcase(path, pattern[3:])) + for pattern in patterns) def inventory(tree: Path, policy: CoveragePolicy) -> list[str]: @@ -289,7 +291,8 @@ def render_contract(full_name: str, pin: str, path: str, raw: str) -> str | None return None body, _ = contract end = len(raw.splitlines()) - ref = f"[完整声明与实现](https://github.com/{full_name}/blob/{pin}/{quote(path, safe='/')}#L1-L{end})" + from .source_links import source_link + ref = f"[完整声明与实现]({source_link(full_name, pin, path, 1, end)})" return f"**`{path}`**\n\n{body}\n\n源码依据：{ref}。" @@ -302,8 +305,8 @@ def contract_block(full_name: str, pin: str, path: str, raw: str) -> str | None: def _citations(text: str, tree: Path, full_name: str, pin: str) -> set[str]: - pattern = re.compile(r"https://github\.com/" + re.escape(full_name) + r"/blob/" + pin - + r"/([^\s)#]+)#L(\d+)(?:-L(\d+))?") + from .source_links import source_base + pattern = re.compile(re.escape(source_base(full_name, pin)) + r"([^\s)#]+)#L(\d+)(?:-L(\d+))?") valid = set() for encoded, start, end in pattern.findall(text): path = unquote(encoded) diff --git a/src/infermatrix_copilot/kb_service/knowledge_depth.py b/src/infermatrix_copilot/kb_service/knowledge_depth.py index 3e4abf99c..809b23585 100644 --- a/src/infermatrix_copilot/kb_service/knowledge_depth.py +++ b/src/infermatrix_copilot/kb_service/knowledge_depth.py @@ -631,9 +631,10 @@ def render_block(feature, section: dict, tree: Path, full_name: str, pin: str, * if proof["trace"]: body += "调用路径：" + " → ".join(chr(96) + s["path"] + chr(96) + "（" + chr(96) + s["symbol"] + chr(96) + "）" for s in proof["trace"]) + "\n\n" + from .source_links import source_link body += "来源：" + ", ".join( - f"[{e['path']}:L{e['start']}–L{e['end']}](https://github.com/{full_name}/blob/{pin}/" - f"{quote(e['path'], safe='/')}#L{e['start']}-L{e['end']})" for e in section["evidence"]) + "\n" + f"[{e['path']}:L{e['start']}–L{e['end']}]({source_link(full_name, pin, e['path'], e['start'], e['end'])})" + for e in section["evidence"]) + "\n" body += "\n" return (f"\n" + body + "\n") diff --git a/src/infermatrix_copilot/kb_service/knowledge_store.py b/src/infermatrix_copilot/kb_service/knowledge_store.py new file mode 100644 index 000000000..c2794b79f --- /dev/null +++ b/src/infermatrix_copilot/kb_service/knowledge_store.py @@ -0,0 +1,329 @@ +"""Canonical knowledge and hash-bound mirrors in the existing snapshot format.""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import tempfile +from pathlib import Path, PurePosixPath + +from .git_source import COMMIT_ID +from .repo_spec import REPO_ID, canonical_json, digest + + +class KnowledgeStoreError(RuntimeError): + pass + + +def _pin(value: str) -> str: + if not COMMIT_ID.fullmatch(value): + raise KnowledgeStoreError("source/publication identity must be a full SHA1 or SHA256 commit") + return value + + +def _path(root: Path, relative: str) -> Path: + pure = PurePosixPath(relative) + target = (root / pure).resolve() + if not relative or pure.is_absolute() or ".." in pure.parts or "\\" in relative or not target.is_relative_to(root): + raise KnowledgeStoreError("knowledge path escapes its store") + return target + + +def _hash_files(root: Path) -> dict[str, str]: + files = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise KnowledgeStoreError("knowledge stores cannot read through symlinks") + if path.is_file(): + files[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return files + + +def immutable_json(path: Path, value: dict) -> str: + data = canonical_json(value) + path.parent.mkdir(parents=True, exist_ok=True) + try: + with path.open("xb") as stream: + stream.write(data) + except FileExistsError: + if path.read_bytes() != data: + raise KnowledgeStoreError("immutable acceptance artifact already exists with different bytes") + return hashlib.sha256(data).hexdigest() + + +def local_acceptance_receipt(*, repo_id: str, source_pin: str, publication_head: str, + files: dict[str, str | bytes], checks: dict, accepted: bool, + review_receipts: tuple[Path, ...] = ()) -> dict: + """Bind a genuinely accepted local publication without fabricating a PR. + + The caller runs the existing source/native/format gates. This receipt + records their evidence; it never creates or rewrites model approvals. + """ + if not REPO_ID.fullmatch(repo_id) or accepted is not True: + raise KnowledgeStoreError("local publication needs explicit acceptance and a stable repo_id") + if not checks or any(value is not True for value in checks.values()): + raise KnowledgeStoreError("local publication checks must all pass; unknown is not acceptance") + refs = [] + for path in review_receipts: + path = Path(path).resolve() + data = path.read_bytes() + refs.append({"path": str(path), "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)}) + hashes = {} + for path, value in sorted(files.items()): + _path(Path("/"), path) + if path.startswith("/") or ".." in PurePosixPath(path).parts: + raise KnowledgeStoreError("accepted file path is invalid") + hashes[path] = hashlib.sha256(value.encode() if isinstance(value, str) else value).hexdigest() + if not hashes: + raise KnowledgeStoreError("an empty local publication cannot be accepted") + return {"schema_version": 1, "kind": "local_knowledge_acceptance", "repo_id": repo_id, + "accepted": True, "forge": "none", "source_pin": _pin(source_pin), + "publication_head": _pin(publication_head), "files": hashes, + "files_sha256": digest(hashes), "checks": checks, "review_receipts": refs, + "claim": "Local accepted publication; source evidence stays at S0, publication is S1. Not a forged PR or new native approval."} + + +def validate_local_acceptance(receipt: dict, *, repo_id: str, source_pin: str, + publication_head: str, files: dict[str, str | bytes]) -> None: + if not isinstance(receipt, dict): + raise KnowledgeStoreError("local acceptance receipt is malformed") + paths = tuple(Path(row["path"]) for row in receipt.get("review_receipts", [])) + expected = local_acceptance_receipt(repo_id=repo_id, source_pin=source_pin, + publication_head=publication_head, files=files, + checks=receipt.get("checks", {}), accepted=receipt.get("accepted"), + review_receipts=paths) + if receipt != expected: + raise KnowledgeStoreError("local acceptance does not bind current files, heads or review artifacts") + + +class KnowledgeStore: + """``canonical_root`` is the full knowledge root in the existing format. + + Central: ``/knowledge``. In-repo: ``/.infermatrix/knowledge``. + Both contain ``repos//`` and share unchanged owner/page schemas. + Read-only mirrors use the normal MANIFEST.json + knowledge/ layout. + """ + + def __init__(self, canonical_root: str | Path, repo_id: str, *, mode: str = "central", readonly: bool = False): + if not REPO_ID.fullmatch(repo_id) or mode not in ("central", "in_repo"): + raise KnowledgeStoreError("invalid canonical knowledge store configuration") + self.root = Path(canonical_root).resolve() + self.repo_id = repo_id + self.mode = mode + self.readonly = readonly or (self.root.parent / "MIRROR.json").exists() + + @property + def knowledge_slice(self) -> str: + return f"repos/{self.repo_id}" + + @property + def acceptance_path(self) -> Path: + return self.root / f"_publication-{self.repo_id}.json" + + def _accepted_scope(self) -> dict: + """Bind this repository and shared procedures, not another repo's tier.""" + hashes = _hash_files(self.root) + files = {p: h for p, h in hashes.items() if p.startswith((self.knowledge_slice + "/", "general/")) + or p in ("AGENTS.md", "_format.yaml")} + registry_path = self.root / "_repositories.yaml" + registry_entry = None + if registry_path.is_file(): + import yaml + registry_entry = (yaml.safe_load(registry_path.read_text()) or {}).get("repos", {}).get(self.repo_id) + return {"files": files, "registry_entry": registry_entry} + + def _validate_acceptance(self, acceptance: dict, source_pin: str) -> None: + if not isinstance(acceptance, dict) or acceptance.get("tier") not in ("final", "foundation"): + raise KnowledgeStoreError("publication acceptance tier must be explicit") + final = acceptance["tier"] == "final" + if acceptance.get("init_complete") is not final or acceptance.get("source_pin") != _pin(source_pin): + raise KnowledgeStoreError("publication acceptance source or completion status differs") + catalog = acceptance.get("catalog_sha256", "") + if not isinstance(catalog, str) or len(catalog) != 64 or any(c not in "0123456789abcdef" for c in catalog): + raise KnowledgeStoreError("publication acceptance needs the exact catalog hash") + checks = acceptance.get("checks") + required = ("source", "native", "structural", "semantic_depth", "retrieval") if final else ("source", "native", "structural") + if not isinstance(checks, dict) or any(checks.get(name) is not True for name in required): + raise KnowledgeStoreError("publication acceptance is missing required passing gates") + + def record_acceptance(self, *, source_pin: str, publication_head: str, acceptance: dict) -> dict: + if self.readonly: + raise KnowledgeStoreError("mirrors cannot record canonical acceptance") + self._validate_acceptance(acceptance, source_pin) + catalog = acceptance["catalog_sha256"] + scope = self._accepted_scope() + spec = scope["registry_entry"] + if spec and (spec.get("source_pin") != source_pin or spec.get("catalog_hash") != catalog): + raise KnowledgeStoreError("publication acceptance differs from the frozen served registry") + record = {"schema_version": 1, "repo_id": self.repo_id, "source_pin": source_pin, + "publication_head": _pin(publication_head), "scope_sha256": digest(scope), "acceptance": acceptance} + self.root.mkdir(parents=True, exist_ok=True) + temporary = self.acceptance_path.with_suffix(".tmp") + temporary.write_bytes(canonical_json(record)) + temporary.replace(self.acceptance_path) + return record + + def _bound_acceptance(self, source_pin: str, publication_head: str) -> dict | None: + if not self.acceptance_path.exists(): + return None + try: + record = json.loads(self.acceptance_path.read_text()) + except (ValueError, OSError) as exc: + raise KnowledgeStoreError("canonical publication acceptance is unreadable") from exc + if (not isinstance(record, dict) or record.get("schema_version") != 1 or record.get("repo_id") != self.repo_id + or record.get("source_pin") != source_pin or record.get("publication_head") != publication_head + or record.get("scope_sha256") != digest(self._accepted_scope())): + raise KnowledgeStoreError("canonical publication acceptance no longer binds these heads/files") + acceptance = record.get("acceptance") + self._validate_acceptance(acceptance, source_pin) + return acceptance + + def write_files(self, files: dict[str, str | bytes], *, expected_hashes: dict[str, str] | None = None) -> None: + if self.readonly: + raise KnowledgeStoreError("mirrors are read-only; publish through the canonical store") + prepared = {} + for relative, value in files.items(): + if not relative.startswith(self.knowledge_slice + "/"): + raise KnowledgeStoreError("canonical publication crosses repository slices") + target = _path(self.root, relative) + if expected_hashes is not None: + expected = expected_hashes.get(relative) + actual = hashlib.sha256(target.read_bytes()).hexdigest() if target.is_file() else "" + if expected is None or expected != actual: + raise KnowledgeStoreError("canonical file changed since review") + prepared[target] = value.encode() if isinstance(value, str) else value + # Validate every destination before writing any member. + for target, data in prepared.items(): + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".kb-tmp") + temporary.write_bytes(data) + temporary.replace(target) + + def identity(self, *, source_pin: str, publication_head: str) -> dict: + if not self.root.is_dir(): + raise KnowledgeStoreError("canonical knowledge root is missing") + files = _hash_files(self.root) + if not files: + raise KnowledgeStoreError("canonical knowledge root is empty") + from ..knowledge_view import SUPPORTED_FORMATS, knowledge_format + if "AGENTS.md" not in files or knowledge_format(self.root) not in SUPPORTED_FORMATS: + raise KnowledgeStoreError("canonical knowledge root lacks the supported existing format") + value = {"repo_id": self.repo_id, "source_pin": _pin(source_pin), + "publication_head": _pin(publication_head), "files": files} + accepted = self._bound_acceptance(source_pin, publication_head) + if accepted is not None: + value["acceptance"] = accepted + return {**value, "snapshot": digest(value)} + + def snapshot(self, destination: str | Path, *, source_pin: str, publication_head: str, + acceptance: dict | None = None) -> dict: + dest = Path(destination).resolve() + if dest == self.root or dest.is_relative_to(self.root) or self.root.is_relative_to(dest): + raise KnowledgeStoreError("snapshot cannot overlap its canonical knowledge root") + if dest.exists(): + raise KnowledgeStoreError("snapshot destination already exists") + if acceptance is not None: + self.record_acceptance(source_pin=source_pin, publication_head=publication_head, acceptance=acceptance) + identity = self.identity(source_pin=source_pin, publication_head=publication_head) + dest.mkdir(parents=True) + shutil.copytree(self.root, dest / "knowledge") + from ..knowledge_view import build_manifest + manifest = {**build_manifest(dest / "knowledge", identity["snapshot"]), + **identity, "knowledge_storage": self.mode} + immutable_json(dest / "MANIFEST.json", manifest) + if _hash_files(dest / "knowledge") != identity["files"]: + raise KnowledgeStoreError("canonical files changed while snapshot was copied") + return manifest + + @staticmethod + def verify_mirror(destination: str | Path) -> dict: + dest = Path(destination).resolve() + try: + manifest = json.loads((dest / "MANIFEST.json").read_text()) + marker = json.loads((dest / "MIRROR.json").read_text()) + except (OSError, ValueError) as exc: + raise KnowledgeStoreError("mirror metadata is missing or unreadable") from exc + if marker.get("readonly") is not True or marker.get("manifest_sha256") != hashlib.sha256((dest / "MANIFEST.json").read_bytes()).hexdigest(): + raise KnowledgeStoreError("mirror manifest binding is invalid") + identity = {key: manifest[key] for key in ("repo_id", "source_pin", "publication_head", "files")} + if "acceptance" in manifest: + identity["acceptance"] = manifest["acceptance"] + if "canonical_snapshot" in manifest: + identity["canonical_snapshot"] = manifest["canonical_snapshot"] + if manifest.get("schema_version") != 1 or digest(identity) != manifest.get("snapshot"): + raise KnowledgeStoreError("mirror snapshot identity is invalid") + if marker.get("repo_id") != manifest["repo_id"] or marker.get("canonical_snapshot") != manifest.get("canonical_snapshot", manifest["snapshot"]): + raise KnowledgeStoreError("mirror authority does not match its canonical snapshot") + if _hash_files(dest / "knowledge") != manifest["files"]: + raise KnowledgeStoreError("mirror differs from its canonical accepted snapshot") + from ..knowledge_view import build_manifest + expected = build_manifest(dest / "knowledge", manifest["snapshot"]) + if any(manifest.get(key) != value for key, value in expected.items()): + raise KnowledgeStoreError("mirror does not preserve the verified knowledge manifest format") + if {p.name for p in dest.iterdir()} != {"knowledge", "MANIFEST.json", "MIRROR.json"}: + raise KnowledgeStoreError("mirror contains unbound files") + return manifest + + def sync_mirror(self, destination: str | Path, *, source_pin: str, publication_head: str, + acceptance: dict | None = None) -> dict: + """Refuse edited/unbound mirrors; only accepted canonical bytes replace one.""" + if self.readonly: + raise KnowledgeStoreError("a mirror cannot become a publication authority") + if acceptance is not None and acceptance != self._bound_acceptance(source_pin, publication_head): + raise KnowledgeStoreError("mirror acceptance must come from the canonical publication") + dest = Path(destination).resolve() + if dest.exists(): + old = self.verify_mirror(dest) + if old["repo_id"] != self.repo_id: + raise KnowledgeStoreError("mirror belongs to another repository") + dest.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="kb-mirror-", dir=dest.parent) as scratch: + prepared = Path(scratch) / "snapshot" + origin = self.identity(source_pin=source_pin, publication_head=publication_head) + copied = prepared / "knowledge" + copied.mkdir(parents=True) + for relative in origin["files"]: + if (relative.startswith((self.knowledge_slice + "/", "general/")) + or relative in ("AGENTS.md", "_format.yaml", self.acceptance_path.name)): + target = _path(copied, relative) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(self.root / relative, target) + registry = self._accepted_scope()["registry_entry"] + if registry is not None: + (copied / "_repositories.yaml").write_bytes(canonical_json({"schema_version": 1, "repos": {self.repo_id: registry}})) + navigation = (f'---\ntitle: "Repositories"\ncreated: 2026-10-06\nupdated: 2026-10-06\ntype: index\ntags: [general]\nsources: []\n---\n\n# Repositories\n\n- [{self.repo_id}]({self.repo_id}/_index.md)\n') + (copied / "repos").mkdir(exist_ok=True) + (copied / "repos/_index.md").write_text(navigation) + # The public subset has its own identity. The full origin remains + # hash-bound without disclosing other repository entries/content. + subset = {"repo_id": self.repo_id, "source_pin": source_pin, "publication_head": publication_head, + "files": _hash_files(copied), "canonical_snapshot": origin["snapshot"]} + if "acceptance" in origin: + subset["acceptance"] = origin["acceptance"] + from ..knowledge_view import build_manifest + manifest = {**build_manifest(copied, digest(subset)), **subset, + "knowledge_storage": self.mode} + immutable_json(prepared / "MANIFEST.json", manifest) + if self.identity(source_pin=source_pin, publication_head=publication_head) != origin: + raise KnowledgeStoreError("canonical files changed while mirror was prepared") + marker = {"schema_version": 1, "repo_id": self.repo_id, "readonly": True, + "canonical_snapshot": origin["snapshot"], + "manifest_sha256": hashlib.sha256((prepared / "MANIFEST.json").read_bytes()).hexdigest()} + immutable_json(prepared / "MIRROR.json", marker) + self.verify_mirror(prepared) + if dest.exists(): + # Check again immediately before replacement to catch edits + # during preparation. The previous accepted mirror is retained + # until the prepared copy has passed its full hash audit. + self.verify_mirror(dest) + backup = Path(scratch) / "previous" + dest.rename(backup) + try: + prepared.rename(dest) + except OSError: + backup.rename(dest) + raise + else: + prepared.rename(dest) + return manifest diff --git a/src/infermatrix_copilot/kb_service/model_dispatch.py b/src/infermatrix_copilot/kb_service/model_dispatch.py new file mode 100644 index 000000000..1b9f738fa --- /dev/null +++ b/src/infermatrix_copilot/kb_service/model_dispatch.py @@ -0,0 +1,89 @@ +"""Host/shared-filesystem dispatch slots for portable initialization. + +Every participating generator and judge uses the same lock directory. This +limits active calls across threads, processes, repositories and batch state +directories. Separate lock directories are independent deployments; this is +not a distributed coordinator. Kernel file locks release on process exit. +""" + +from __future__ import annotations + +import fcntl +import json +import logging +import os +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) +MAX_CONCURRENCY = 13 + + +class DispatchConfigError(ValueError): + pass + + +@dataclass(frozen=True) +class DispatchTicket: + slot: int + wait_seconds: float + limit: int + directory: Path + + +class SharedModelDispatch: + def __init__(self, directory: str | Path | None = None, *, limit: int | None = None, + environ=None): + env = os.environ if environ is None else environ + raw = env.get("KB_INIT_GLOBAL_CONCURRENCY", "13") if limit is None else limit + if isinstance(raw, bool) or not str(raw).isdigit() or not 1 <= int(raw) <= MAX_CONCURRENCY: + raise DispatchConfigError("KB_INIT_GLOBAL_CONCURRENCY must be an integer from 1 to 13") + self.limit = int(raw) + cache = Path(env.get("XDG_CACHE_HOME") or Path.home() / ".cache") + self.directory = Path(directory or env.get("KB_INIT_DISPATCH_DIR") or + cache / "infermatrix-copilot" / "kb-init-dispatch").expanduser().resolve() + self.directory.mkdir(parents=True, exist_ok=True) + # One namespace has one declared cap. Conflicting caps cannot silently + # produce distinct pools on a shared host; choose an explicit namespace + # when configuring an independent deployment. + with (self.directory / "configuration.lock").open("a+b") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + path = self.directory / "configuration.json" + expected = {"schema_version": 1, "limit": self.limit} + if path.exists(): + try: + actual = json.loads(path.read_text()) + except (ValueError, OSError) as exc: + raise DispatchConfigError("shared model dispatch configuration is unreadable") from exc + if actual != expected: + raise DispatchConfigError("shared model dispatch cap conflicts with this namespace") + else: + with path.open("x") as stream: + json.dump(expected, stream, sort_keys=True) + + @contextmanager + def slot(self): + started = time.monotonic() + last_notice = started + while True: + for index in range(self.limit): + stream = (self.directory / f"slot-{index:02d}.lock").open("a+b") + try: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + stream.close() + continue + try: + yield DispatchTicket(index, time.monotonic() - started, self.limit, self.directory) + finally: + fcntl.flock(stream, fcntl.LOCK_UN) + stream.close() + return + now = time.monotonic() + if now - last_notice >= 30: + logger.info("Waiting for shared portable model dispatch slot (cap %s, %.1fs)", + self.limit, now - started) + last_notice = now + time.sleep(0.1) diff --git a/src/infermatrix_copilot/kb_service/models.py b/src/infermatrix_copilot/kb_service/models.py index 0d52e94ab..c3b0a68b4 100644 --- a/src/infermatrix_copilot/kb_service/models.py +++ b/src/infermatrix_copilot/kb_service/models.py @@ -125,6 +125,11 @@ def __init__(self, settings, *, transport_factory: Callable[[str], Any] | None = self._recorder = recorder self._subscription_transports: dict[str, Any] = {} self._zcode_pacer = zcode_pacer + self._dispatch = None + + def configure_dispatch(self, dispatch): + """Portable initialization opts into one shared native dispatch cap.""" + self._dispatch = dispatch def configure_zcode_pacing(self, pacer): """Only the explicit native campaign configures shared dispatch pacing.""" @@ -179,6 +184,18 @@ def _call_json(self, role: ModelRole, *, system: str, prompt: str, validate: Callable[[dict], None] | None = None, max_budget_usd: float | None = None, record_payload: bool = True, fallback_from: str = "") -> ModelReply: + kwargs = dict(system=system, prompt=prompt, validate=validate, + max_budget_usd=max_budget_usd, record_payload=record_payload, + fallback_from=fallback_from) + if self._dispatch is None: + return self._call_json_native(role, **kwargs) + with self._dispatch.slot(): + return self._call_json_native(role, **kwargs) + + def _call_json_native(self, role: ModelRole, *, system: str, prompt: str, + validate: Callable[[dict], None] | None = None, + max_budget_usd: float | None = None, record_payload: bool = True, + fallback_from: str = "") -> ModelReply: """``max_budget_usd`` is a per-call STOP THRESHOLD, not a hard cap: the transport starts no further API request once the call's spend reaches it, but the request that crosses it is billed in full. A diff --git a/src/infermatrix_copilot/kb_service/native_depth_audit.py b/src/infermatrix_copilot/kb_service/native_depth_audit.py new file mode 100644 index 000000000..3f3e8dfb5 --- /dev/null +++ b/src/infermatrix_copilot/kb_service/native_depth_audit.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Verify current depth approvals against archived native model calls and blobs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from bisect import bisect_right +from pathlib import Path +from urllib.parse import quote + +import yaml + +from infermatrix_copilot.kb_service.gate import DIMENSIONS +from infermatrix_copilot.kb_service.source_links import source_link +from infermatrix_copilot.kb_service.init_stages import neutral_headings +from infermatrix_copilot.kb_service.knowledge_coverage import load_policy, policy_path +from infermatrix_copilot.kb_service.knowledge_depth import depth_page, depth_acceptance_mode, validate_draft +from infermatrix_copilot.kb_service.models import ModelUnavailable, parse_json_object +from infermatrix_copilot.knowledge_service.lifecycle import ( + DEPTH_BLOCK, DEPTH_FACETS, depth_proof_basis, depth_sections, safe_source_path, +) +from infermatrix_copilot.trace_store import TraceStore + +MODEL_FIELDS = ("role", "provider", "model", "effort", "served_model") + + +def _sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _payload(text: str) -> dict: + match = re.fullmatch(r"\s*(.*?)\s*\s*", text, re.S) + data = json.loads(match[1] if match else text) + if not isinstance(data, dict): + raise ValueError("native prompt must contain an object") + return data + + +def _prose(block: str) -> str: + # This is exactly the rendering used by depth_judge.review_facets. + return re.sub(r"", "", block, flags=re.S).strip() + + +def _family(model: str) -> str: + return model.casefold().split("-")[0] + + +def _generated_prose(section: dict, repository: str, pin: str) -> str: + """Replay only render_block's deterministic lightweight prose transforms.""" + validate_draft({"sections": [section]}, acceptance_mode="lightweight") + title = " ".join(str(section.get("title") or section["facet"]).replace("#", "").split()) + body = neutral_headings(f"## {title}\n\n") + if section.get("validation_kind") == "documented_manual": + body += "文档中的人工验收步骤（本轮未执行）：\n\n" + if section["interpretation"] == "inference": + body += "设计推断（非作者历史意图）：\n\n" + body += section["body"].strip() + "\n\n" + body += "来源：" + ", ".join( + f"[{e['path']}:L{e['start']}–L{e['end']}]({source_link(repository, pin, e['path'], e['start'], e['end'])})" for e in section["evidence"]) + "\n" + return _prose(body) + + +def _generator_evidence(data: dict, repository: str, pin: str) -> dict: + """Convert the actual offered files/docs to the existing exact-span verifier.""" + packet = [] + for kind in ("files", "docs"): + items = data.get(kind, []) + if not isinstance(items, list): + raise ValueError("native generator offered spans are malformed") + for item in items: + if not isinstance(item, dict) or not safe_source_path(item.get("path")) \ + or type(item.get("start")) is not int or type(item.get("end")) is not int \ + or not 1 <= item["start"] <= item["end"] \ + or not isinstance(item.get("text"), list) \ + or len(item["text"]) != item["end"] - item["start"] + 1: + raise ValueError("native generator offered spans are malformed") + rows = item["text"] if kind == "files" else [f"{n}: {line}" for n, line in enumerate(item["text"], item["start"])] + if any(not isinstance(line, str) for line in item["text"]): + raise ValueError("native generator offered lines are malformed") + packet.append({"kind": "upstream_text", "text": rows, + "source_reference": f"{repository}@{pin}:{item['path']}:L{item['start']}-L{item['end']}"}) + return {"evidence": packet} + + +def _lightweight_generator(candidates, records, *, before: float, feature: str, facet: str, + block: str, proof: dict, repository: str, pin: str) -> dict: + """Find the actual retained draft, which may precede a later unrelated retry.""" + expected = [{key: e[key] for key in ("path", "start", "end")} for e in proof["evidence"]] + for candidate in reversed(candidates): + native = records.get(candidate["id"]) + if candidate["at"] > before or native is None: + continue + record, store = native + model = record.get("model", {}) + requested, served = model.get("model"), model.get("served_model") + if model.get("provider") != "zcode" or not isinstance(requested, str) or requested.casefold() != "glm-5.3" \ + or model.get("fallback_from") or record.get("result", {}).get("fallback_from") \ + or served not in (None, "") and (not isinstance(served, str) or served.casefold() != "glm-5.3"): + continue + try: + data = _payload(store.blob(record["inputs"]["prompt"])) + requested_facets = data.get("facets") + if data.get("repository") != repository or data.get("feature", {}).get("id") != feature \ + or data.get("pin") != pin or data.get("acceptance_mode") != "lightweight" \ + or not isinstance(requested_facets, list) or requested_facets.count(facet) != 1: + continue + draft = parse_json_object(store.blob(record["outputs"]["reply"])) + sections = draft.get("sections") + if not isinstance(sections, list): + continue + matching = [section for section in sections if isinstance(section, dict) and section.get("facet") == facet] + if len(matching) != 1: + continue + section = matching[0] + spans = [{key: e[key] for key in ("path", "start", "end")} for e in section["evidence"]] + if spans != expected or section.get("validation_kind") != proof.get("validation_kind") \ + or _generated_prose(section, repository, pin) != _prose(block): + continue + _judged_evidence(_generator_evidence(data, repository, pin), block, repository, pin, facet) + return record + except (OSError, ValueError, TypeError, KeyError, AttributeError, ModelUnavailable): + continue + raise ValueError("no exact successful Zcode GLM-5.3 lightweight draft and offered evidence before this judgment") + + +def _judged_evidence(data: dict, block: str, repository: str, pin: str, facet: str) -> None: + """Bind the actual judge context to each persisted source witness. + + Historical packets merge adjoining evidence spans and omit line endings. + Source-span hashes normalize CRLF through read_text, so replay their UTF-8 + text with either a final LF or no terminator for a source's final line. + """ + match = re.search(r"\n\s*\n$", block, re.S) + if match is None: + raise ValueError("current facet has no evidence proof") + proof = json.loads(match[1]) + basis = depth_proof_basis(proof, facet=facet, pin=pin) + evidence = data.get("evidence") + if not isinstance(evidence, list) or not evidence: + raise ValueError("native judgment has no source evidence packet") + lines, certificates = {}, [] + for item in evidence: + if not isinstance(item, dict): + raise ValueError("native judgment has malformed evidence") + if item.get("kind") == "replayed_absence_certificate": + certificates.append(item.get("certificate")) + continue + if item.get("kind") != "upstream_text": + raise ValueError("native judgment has an unknown evidence kind") + reference = item.get("source_reference") + prefix = repository + "@" + pin + ":" + if not isinstance(reference, str) or not reference.startswith(prefix): + raise ValueError("native evidence reference differs from the repository or pin") + span = re.fullmatch(r"(.+):L([1-9][0-9]*)-L([1-9][0-9]*)", reference[len(prefix):]) + if span is None or not safe_source_path(span[1]): + raise ValueError("native evidence has an invalid source reference") + path, start, end = span[1], int(span[2]), int(span[3]) + text = item.get("text") + if end < start or not isinstance(text, list) or len(text) != end - start + 1: + raise ValueError("native numbered evidence does not cover its declared span") + for number, row in enumerate(text, start): + marker = f"{number}: " + if not isinstance(row, str) or not row.startswith(marker) or "\n" in row or "\r" in row: + raise ValueError("native numbered evidence is malformed or noncontiguous") + value = row[len(marker):] + key = (path, number) + if key in lines and lines[key] != value: + raise ValueError("native evidence contains conflicting source lines") + lines[key] = value + for span in proof["evidence"]: + try: + shown = "\n".join(lines[(span["path"], number)] for number in range(span["start"], span["end"] + 1)) + except KeyError as exc: + raise ValueError("native judgment omitted a current proof span") from exc + if span["sha256"] not in {_sha(shown.encode()), _sha((shown + "\n").encode())}: + raise ValueError("native source text differs from its current proof hash") + if basis == "verified_absent" and certificates != [proof["absence_certificate"]]: + raise ValueError("native judgment omitted or changed the replayed absence certificate") + + +def _archives(paths: list[Path]) -> tuple[dict, dict, list[dict], list[str]]: + records, generators, provenance, problems = {}, {}, [], [] + for root in paths: + store = TraceStore(root) + record_files = sorted((root / "records").glob("*.jsonl")) + if not record_files: + problems.append(f"{root.name}: no native trace record archive") + continue + record_bytes = {path: path.read_bytes() for path in record_files} + hashed = hashlib.sha256() + blob_files = sorted((root / "blobs").rglob("*.gz")) + for path in sorted(record_files + blob_files): + raw = record_bytes[path] if path in record_bytes else path.read_bytes() + hashed.update(path.relative_to(root).as_posix().encode() + b"\0" + raw) + provenance.append({"name": root.name, "archive_sha256": hashed.hexdigest(), + "record_files": len(record_files), "blob_files": len(blob_files)}) + for path in record_files: + for number, line in enumerate(record_bytes[path].decode().splitlines(), 1): + try: + record = json.loads(line) + if not isinstance(record, dict) or record.get("schema") != "trace/1" \ + or not isinstance(record.get("id"), str) or not record["id"]: + raise ValueError("invalid native trace record") + record_id = record["id"] + if record_id in records: + problems.append(f"duplicate native trace ID: {record_id}") + records[record_id] = None + continue + records[record_id] = (record, store) + if record.get("kind") != "model_call" or record.get("error") != "" \ + or record.get("model", {}).get("role") != "generator": + continue + data = _payload(store.blob(record["inputs"]["prompt"])) + feature = data.get("feature") + feature = feature.get("id") if isinstance(feature, dict) else feature + pin = data.get("pin") + if isinstance(feature, str) and isinstance(pin, str) \ + and isinstance(record.get("at"), (int, float)): + generators.setdefault((feature, pin), []).append(record) + except (OSError, ValueError, TypeError, KeyError, AttributeError) as exc: + problems.append(f"{root.name}/{path.name}:{number}: unreadable native record or blob ({type(exc).__name__})") + for rows in generators.values(): + rows.sort(key=lambda row: (row["at"], row["id"])) + return records, generators, provenance, problems + + +def _receipts(baselines: list[Path], checkpoints: list[Path], pages: dict) -> tuple[dict, list[dict], list[str]]: + receipts, provenance, problems = {}, [], [] + for kind, paths in (("baseline_report", baselines), ("checkpoint", checkpoints)): + for path in paths: + try: + raw = path.read_bytes() + data = json.loads(raw) + provenance.append({"kind": kind, "name": path.name, "sha256": _sha(raw)}) + if kind == "baseline_report": + rows = data.get("approvals", data.get("native_execution", {}).get("approvals")) + if not isinstance(rows, list): + raise ValueError("baseline report must contain an approvals list") + else: + rows = [] + for key, verdict in data.get("verdicts", {}).items(): + if not key.startswith("depth:"): + continue + feature = key.removeprefix("depth:") + for facet, result in verdict.get("facets", {}).items(): + if result.get("verdict") == "pass": + rows.append({**result, "feature": feature, "facet": facet, + "page": pages.get(feature), "pin": data.get("pin")}) + for row in rows: + if not isinstance(row, dict) or not isinstance(row.get("feature"), str) \ + or row.get("facet") not in DEPTH_FACETS: + raise ValueError("malformed native approval receipt") + key = (row["feature"], row["facet"]) + if key in receipts: + problems.append("duplicate native approval receipt: " + "/".join(key)) + receipts[key] = None + else: + receipts[key] = row + except (OSError, ValueError, TypeError, AttributeError) as exc: + problems.append(f"{path.name}: unreadable {kind} ({type(exc).__name__})") + return receipts, provenance, problems + + +def audit_native_approvals(root: Path, repo: str, *, baseline_reports: list[Path], + trace_dirs: list[Path], checkpoints: list[Path]) -> dict: + """No model calls or archive/index writes; reports contain no raw payloads. + + This verifies native judgment bindings. The separate pinned depth auditor + owns source-span, flow and absence-certificate replay against upstream. + """ + root = root.resolve() + if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", repo): + raise ValueError("repo must be a canonical adapter slug") + manifest = yaml.safe_load((root / f"adapters/{repo}/manifest.yaml").read_text()) + repository = manifest.get("repo", {}).get("full_name") + if not isinstance(repository, str) or not repository or any(c in repository for c in ("\n", "\r", "\0")): + raise ValueError("adapter must declare its source repository full_name") + policy_file = root / policy_path(repo) + policy = load_policy(policy_file.read_text(), manifest["knowledge"]["repo_subdir"]) + pages = {feature.id: depth_page(feature) for feature in policy.features} + records, generators, archive_provenance, problems = _archives(trace_dirs) + receipts, receipt_provenance, receipt_problems = _receipts(baseline_reports, checkpoints, pages) + problems += receipt_problems + approvals, current_keys, knowledge_hash = [], set(), hashlib.sha256() + dimensions = {key: "yes" for key in DIMENSIONS["prose"]} + for feature in policy.features: + page = pages[feature.id] + path = root / "knowledge" / page + if not path.is_file(): + continue + text = path.read_text() + knowledge_hash.update(page.encode() + b"\0" + text.encode()) + matches = list(DEPTH_BLOCK.finditer(text)) + valid = {(row["feature"], row["facet"]) for row in depth_sections(text)} + if text.count("", match[0], re.S)[1]) + mode = depth_acceptance_mode(proof) + receipt = receipts.get(key) + if not receipt or receipt.get("page") != page or receipt.get("block_sha256") != block_hash \ + or receipt.get("pin", pin) != pin: + raise ValueError("missing or mismatched current-block approval receipt") + if receipt.get("acceptance_mode", "strict") != mode: + raise ValueError("approval receipt acceptance mode differs from current block") + native = records.get(receipt.get("native_trace_id")) + if not native: + raise ValueError("missing or duplicate native judgment trace") + record, store = native + model = record.get("model", {}) + if record.get("kind") != "model_call" or record.get("error") != "" \ + or model.get("role") != "judge" or model.get("provider") != "codex" \ + or not isinstance(model.get("model"), str) or not model["model"]: + raise ValueError("native record is not a successful Codex judge call") + served = model.get("served_model") + if served not in (None, "") and (not isinstance(served, str) + or not re.match(r"^(?:gpt-|codex(?:-|$)|o[0-9]+(?:-|$))", served, re.I)): + raise ValueError("native record reports an incompatible Codex judge served identity") + reply_ref = record.get("outputs", {}).get("reply") + expected_ref = "sha256:" + str(receipt.get("native_reply_sha256", "")) + if reply_ref != expected_ref: + raise ValueError("native reply hash differs from approval receipt") + response = parse_json_object(store.blob(reply_ref)) + result = response.get("facets", {}).get(facet) + if not isinstance(result, dict) or result.get("dimensions") != dimensions \ + or not isinstance(result.get("reason"), str): + raise ValueError("native facet judgment is not exactly three yes dimensions") + data = _payload(store.blob(record["inputs"]["prompt"])) + if data.get("feature") != fid or data.get("pin") != pin \ + or data.get("sections", {}).get(facet) != _prose(match[0]): + raise ValueError("native judgment prompt does not bind the exact current facet prose and pin") + modes = data.get("acceptance_modes", {}) + if not isinstance(modes, dict) or modes.get(facet, "strict") != mode: + raise ValueError("native judgment prompt does not bind the current acceptance mode") + validation_kind = proof.get("validation_kind") + if validation_kind is not None and (receipt.get("validation_kind") != validation_kind or + data.get("validation_kinds", {}).get(facet) != validation_kind): + raise ValueError("native judgment and receipt do not bind the validation category") + _judged_evidence(data, match[0], repository, pin, facet) + sections, results = data["sections"], response["facets"] + if mode == "strict" and (set(sections) != set(results) or any( + name not in DEPTH_FACETS or not isinstance(item, dict) + or not isinstance(item.get("reason"), str) or not isinstance(item.get("dimensions"), dict) + or set(item["dimensions"]) != set(dimensions) + or any(value not in ("yes", "no", "unsure") for value in item["dimensions"].values()) + for name, item in results.items())): + raise ValueError("native response does not answer the complete judged packet") + candidates = generators.get((fid, pin), []) + index = bisect_right([row["at"] for row in candidates], record["at"]) - 1 + if index < 0: + raise ValueError("no successful native generator context before this judgment") + generator = _lightweight_generator(candidates, records, before=record["at"], feature=fid, + facet=facet, block=match[0], proof=proof, repository=repository, pin=pin) \ + if mode == "lightweight" else candidates[index] + if records.get(generator["id"]) is None: + raise ValueError("generator trace ID is duplicated") + generator_model = generator.get("model", {}) + gen_name = generator_model.get("served_model") or generator_model.get("model") + judge_name = model.get("served_model") or model["model"] + if not isinstance(gen_name, str) or not gen_name or not isinstance(judge_name, str) \ + or not isinstance(generator_model.get("provider"), str) or not generator_model["provider"] \ + or _family(gen_name) == _family(judge_name): + raise ValueError("native generator and judge are not independently identified model families") + approvals.append({"feature": fid, "facet": facet, "page": page, "pin": pin, + "acceptance_mode": mode, + **({"validation_kind": validation_kind} if validation_kind is not None else {}), + "block_sha256": block_hash, "dimensions": result["dimensions"], + "native_trace_id": record["id"], "native_reply_sha256": reply_ref.removeprefix("sha256:"), + "model": {key: model.get(key, "") for key in MODEL_FIELDS}, + "native_generator_trace_id": generator["id"], + "generator_model": {key: generator_model.get(key, "") for key in MODEL_FIELDS}}) + except (OSError, ValueError, TypeError, KeyError, AttributeError, ModelUnavailable) as exc: + detail = str(exc) if isinstance(exc, ValueError) else type(exc).__name__ + problems.append(label + ": " + detail) + for key in sorted(set(receipts) - current_keys): + problems.append("approval receipt has no current depth block: " + "/".join(key)) + approvals.sort(key=lambda row: (row["feature"], DEPTH_FACETS.index(row["facet"]))) + return {"schema_version": 1, "repo": repo, "approvals": approvals, "problems": problems, + "provenance": {"policy_sha256": _sha(policy_file.read_bytes()), + "depth_pages_sha256": knowledge_hash.hexdigest(), "trace_archives": archive_provenance, + "input_reports": receipt_provenance, + "approval_set_sha256": _sha(json.dumps(approvals, ensure_ascii=False, sort_keys=True).encode())}, + "interpretation": "Three yes dimensions were read from native reply blobs bound to current prose. " + "Unreported served-model identities remain unreported; requested identity and provider " + "come from the native record. Lightweight generator provenance also binds an exact " + "Zcode GLM-5.3 draft and its actually offered source/document spans without fallback; " + "strict historical provenance retains its existing model-family check. " + "Source and absence proofs require the pinned depth audit."} + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--repo", required=True) + parser.add_argument("--baseline-report", type=Path, action="append", default=[]) + parser.add_argument("--trace-dir", type=Path, action="append", required=True) + parser.add_argument("--checkpoint", type=Path, action="append", default=[]) + parser.add_argument("--report", type=Path) + args = parser.parse_args(argv) + if args.report and args.report.resolve().is_relative_to((args.root / "knowledge").resolve()): + parser.error("approval reports belong outside product knowledge") + try: + report = audit_native_approvals(args.root, args.repo, baseline_reports=args.baseline_report, + trace_dirs=args.trace_dir, checkpoints=args.checkpoint) + except (OSError, ValueError, TypeError, KeyError) as exc: + parser.error(f"cannot read native approval inputs: {type(exc).__name__}") + raw = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(raw) + print(json.dumps({"approvals": len(report["approvals"]), "problems": report["problems"], + "report_sha256": _sha(raw.encode())}, ensure_ascii=False, indent=2)) + return 1 if report["problems"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/infermatrix_copilot/kb_service/portable_commands.py b/src/infermatrix_copilot/kb_service/portable_commands.py new file mode 100644 index 000000000..9c8c1cb0b --- /dev/null +++ b/src/infermatrix_copilot/kb_service/portable_commands.py @@ -0,0 +1,429 @@ +"""Portable CLI orchestration. Reviewed previews and activation are explicit operations.""" +from __future__ import annotations + +import hashlib +import json +import shutil +import tempfile +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path + +import yaml + +from .git_source import GitSource +from .init_support import InitError, InitRecord +from .portable_init import portable_runtime, bootstrap_workspace, stage_receipt, accept_stage, _git +from .repo_spec import RepoRegistry, RepoSpec, acceptance_receipt, propose_repository, repository_inventory, REGISTRY_FILE + + +def add_commands(sub): + onboard = sub.add_parser("onboard", help="propose a portable repository identity and scanning scope") + onboard.add_argument("source") + onboard.add_argument("--propose", action="store_true", default=True) + onboard.add_argument("--repo-id") + onboard.add_argument("--pin") + onboard.add_argument("--storage", choices=("central", "in_repo"), default="central") + onboard.add_argument("--forge", choices=("none", "github", "gitlab")) + onboard.add_argument("--out", type=Path) + repo = sub.add_parser("repo", help="register reviewed proposals and accept local stage previews") + commands = repo.add_subparsers(dest="repo_command", required=True) + review = commands.add_parser("receipt", help="record explicit acceptance of an exact repository proposal") + review.add_argument("--proposal", type=Path, required=True) + review.add_argument("--source", type=Path, required=True) + review.add_argument("--out", type=Path) + register = commands.add_parser("register") + register.add_argument("--proposal", type=Path, required=True) + register.add_argument("--review-receipt", type=Path, required=True) + register.add_argument("--source", type=Path, required=True) + register.add_argument("--knowledge-root", type=Path, required=True) + commands.add_parser("list") + for name in ("review-stage", "accept-stage"): + cmd = commands.add_parser(name) + cmd.add_argument("repo") + cmd.add_argument("--stage", required=True) + if name == "review-stage": + cmd.add_argument("--reviewer", required=True) + cmd.add_argument("--out", type=Path) + else: + cmd.add_argument("--review-receipt", type=Path, required=True) + publish = commands.add_parser("publish", help="publish an accepted knowledge slice to its canonical store") + publish.add_argument("repo") + publish.add_argument("--snapshot-out", type=Path, required=True) + publish.add_argument("--partial-foundation", action="store_true") + mirror = sub.add_parser("mirror", help="export a hash-bound, read-only knowledge snapshot") + mirror.add_argument("repo") + mirror.add_argument("--out", type=Path, required=True) + update = sub.add_parser("update", help="compare the fixed baseline and prepare an incremental discovery batch") + update.add_argument("repo") + update.add_argument("--to", required=True) + update.add_argument("--dry-run", action="store_true") + update.add_argument("--apply", action="store_true", help="register and prepare the new batch; extraction is a separate kb init invocation") + update.add_argument("--out", type=Path) + + +def _json(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def emit(value, path=None): + text = json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if path: + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_text(text, encoding="utf-8") + print(text, end="") + + +def run_portable_init(args, state_dir): + from ..config import Settings + from .init_stages import run_stage + + rt, lc = portable_runtime(Settings(), state_dir, args.repo) + rt.discovery_paths = tuple(_json(Path(state_dir) / "incremental.json").get("scan_paths", [])) \ + if (Path(state_dir) / "incremental.json").is_file() else None + if args.suggest_seeds: + from .init_stages import suggest_seeds + emit({"seeds": suggest_seeds(rt, lc)}) + return 0 + if not args.stage: + raise InitError("kb init requires --stage") + if args.pin and args.pin != rt.portable_spec.source_pin: + raise InitError("source pin differs from registered batch; use kb update to prepare a new batch") + previous = InitRecord.load(Path(state_dir), lc.repo, args.stage) + if previous and previous.status == "accepted": + from .portable_init import check_local_acceptance + check_local_acceptance(rt, previous) + emit({"repo_id": lc.repo, "stage": args.stage, "status": "accepted", "source_pin": previous.pin, + "published": False, "review_required": False}) + return 0 + kwargs = {key: getattr(args, key) for key in ("pr_count", "budget_usd", "from_existing", "subscription_generator", "unlimited_subscription", "retry_unfinished", "acceptance_mode", "foundation_mode") if hasattr(args, key)} + if rt.generator.provider == "zcode": + kwargs["subscription_generator"] = True + if args.stage in ("feature-discovery", "modules", "knowledge", "knowledge-deepen") and kwargs.get("budget_usd") is None: + kwargs["unlimited_subscription"] = True + if args.stage == "knowledge-deepen" and not getattr(args, "acceptance_mode_explicit", False): + kwargs["acceptance_mode"] = "lightweight" + if args.stage == "knowledge": + # Portable foundation generates a new native batch, then offers the + # checked partial preview. The legacy publication-only path is retained. + kwargs["foundation_mode"] = "strict" + elif args.stage == "knowledge-deepen": + kwargs["foundation_mode"] = "partial" + if getattr(args, "foundation_record", None): + kwargs["foundation_record_path"] = args.foundation_record + record = run_stage(rt, lc, args.stage, dry_run=True, pin=args.pin or rt.portable_spec.source_pin, **kwargs) + emit({"repo_id": lc.repo, "stage": args.stage, "status": record.status, + "source_pin": record.pin, "review_required": record.status == "dry_run", + "published": False, "problems": record.problems, "preview": record.pr.get("dry_run_dir")}) + return 0 if record.status in ("dry_run", "accepted", "empty") else 1 + + +def publish_portable(state_dir, selector, dest, *, partial=False): + from .knowledge_store import KnowledgeStore + from .portable_init import check_local_acceptance, LocalKnowledgeRepo, source_identity + from types import SimpleNamespace + + row = RepoRegistry(state_dir).resolve(selector) + if not row: + raise InitError("repository is not registered") + spec = RepoSpec.from_dict(row["spec"]) + stage = "knowledge" if partial else "knowledge-deepen" + record = InitRecord.load(Path(state_dir), spec.repo_id, stage) + if not record or record.status != "accepted": + raise InitError("publication requires an explicitly accepted knowledge preview") + if not partial and not record.depth.get("target_met"): + raise InitError("seven-facet final acceptance is unmet; retain the checkpoint") + root = bootstrap_workspace(state_dir, row) + knowledge = LocalKnowledgeRepo(root) + check_local_acceptance(SimpleNamespace(knowledge=knowledge), record) + head = record.pr["local_acceptance"]["head_sha"] + if knowledge.main_sha() != head: + raise InitError("publication baseline contains changes after the accepted knowledge head") + files = knowledge.knowledge_files(head) + policy_text = knowledge.show(head, f"adapters/{spec.repo_id}/knowledge-coverage.yaml") + if not policy_text: + raise InitError("publication is missing the frozen feature catalog") + catalog = hashlib.sha256(policy_text.encode()).hexdigest() + published_spec = replace(spec, catalog_hash=catalog, policy_hash=catalog) + canonical = Path(row["bindings"]["knowledge_root"]) + dest = Path(dest).resolve() + canonical = canonical.resolve() + if KnowledgeStore(canonical, spec.repo_id, mode=spec.knowledge_storage).readonly: + raise InitError("a read-only mirror cannot be a canonical publication authority") + if dest.exists() or dest == canonical or dest.is_relative_to(canonical) or canonical.is_relative_to(dest): + raise InitError("snapshot destination exists or overlaps the canonical knowledge store") + from .init_support import run_knowledge_validators + problems = run_knowledge_validators(knowledge, head, {}) + if problems: + raise InitError("publication format validation failed: " + "; ".join(problems[:3])) + if partial: + from ..config import Settings + from .portable_init import portable_runtime + from .init_knowledge import _Knowledge + from .init_knowledge_parallel import validate_checkpoint + runtime, lifecycle = portable_runtime(Settings(), state_dir, spec.repo_id) + runtime.unlimited_subscription = True + runtime.subscription_generator = True + if not record.coverage.get("foundation_jobs", {}).get("tasks"): + raise InitError("foundation publication requires replayable native task receipts") + stage_impl = _Knowledge(runtime, lifecycle, dry_run=True, pin=record.pin) + stage_impl.repo_dir = lifecycle.knowledge_dir + stage_impl._base_sha = record.kb_base_sha + stage_impl._knowledge_run_pin = record.pin + stage_impl.overlay = {} + validate_checkpoint(stage_impl, record, record.inputs_digest) + with tempfile.TemporaryDirectory(prefix="kb-publish-source-") as tmp: + tree = GitSource(row["bindings"]["source_path"]).export(spec.source_pin, Path(tmp) / "source") + from .knowledge_coverage import load_policy, audit_coverage, coverage_targets_met, inventory as code_inventory + from .knowledge_depth import audit_depth + policy = load_policy(policy_text, spec.knowledge_slice) + if (set(policy.roots) != set(spec.source_roots) or policy.target < spec.coverage_target + or policy.semantic_depth_per_facet_gt is None + or policy.semantic_depth_per_facet_gt < spec.per_facet_gt): + raise InitError("catalog scope or acceptance policy differs from the reviewed repository proposal") + # The same deterministic empty-Python-marker rule applies to both + # inventories. No catalog suffix/exclusion may hide other production code. + from dataclasses import replace as replace_policy + declared = repository_inventory(GitSource(row["bindings"]["source_path"]), spec)["production"] + eligibility = replace_policy(policy, roots=(".",), exclude=(), + suffixes=tuple(sorted({Path(p).suffix for p in declared if Path(p).suffix})), + filenames=tuple(sorted({Path(p).name for p in declared if not Path(p).suffix}))) + expected_production = set(code_inventory(tree, eligibility)) & set(declared) + if set(code_inventory(tree, policy)) != expected_production: + raise InitError("catalog production inventory differs from the reviewed fixed source scope") + structural = audit_coverage(files, tree, policy, full_name=source_identity(spec), pin=spec.source_pin) + if not coverage_targets_met(structural, "partial"): + raise InitError("production-file structural coverage is unmet") + if not partial: + from .native_depth_audit import audit_native_approvals + native = audit_native_approvals(root, spec.repo_id, baseline_reports=[], + trace_dirs=[Path(state_dir) / "init/traces"], + checkpoints=[InitRecord.path(Path(state_dir), spec.repo_id, "knowledge-deepen")]) + if native["problems"]: + raise InitError("native approval replay failed: " + "; ".join(native["problems"][:3])) + depth = audit_depth(files, tree, policy, spec.source_pin, approvals=native["approvals"]) + if not depth["target_met"]: + raise InitError("current source/knowledge/native audit fails final acceptance") + selected = {p: t for p, t in files.items() if p.startswith(spec.knowledge_slice + "/")} + selected[f"{spec.knowledge_slice}/_catalog.yaml"] = policy_text + discovery_record = InitRecord.load(Path(state_dir), spec.repo_id, "feature-discovery") + if discovery_record: + report = json.loads(knowledge.show(head, discovery_record.discovery["report_path"])) + selected[f"{spec.knowledge_slice}/_discovery.yaml"] = yaml.safe_dump({key: report[key] + for key in ("pin", "catalog_sha256", "index_sha256", "evidence_bundles", "scan_mode", "scan_paths") if key in report}, sort_keys=False) + marker_path = root / ".git" / "kb-registration.json" + marker = _json(marker_path) + expected = {p: marker.get("canonical_files", {}).get(p, "") for p in selected} + acceptance = {"tier": "foundation" if partial else "final", "init_complete": not partial, + "source_pin": spec.source_pin, "catalog_sha256": catalog, + "checks": {"source": True, "native": True, "structural": True, "format": True, + "retrieval": True, "semantic_depth": not partial}, + "structural": structural, "semantic_depth": {} if partial else depth, + "unknown": record.coverage.get("foundation", {}) if partial else {}, + "native_provenance_sha256": None if partial else hashlib.sha256(json.dumps(native["provenance"], sort_keys=True).encode()).hexdigest(), + "actual_cost": "unknown", "upstream_tests_executed": False} + # Validate the staged full store and its served retrieval before replacing + # canonical files. A refused export must not partly publish knowledge. + canonical.parent.mkdir(parents=True, exist_ok=True) + dest.parent.mkdir(parents=True, exist_ok=True) + from .portable_publication import check_publication + from .activate import verify_snapshot + from ..knowledge_docs import KnowledgeDocs + with _canonical_lock(canonical): + if dest.exists(): + raise InitError("snapshot destination already exists") + with tempfile.TemporaryDirectory(prefix=".kb-publish-", dir=canonical.parent) as scratch: + staged = Path(scratch) / "knowledge" + if canonical.exists(): + if any(p.is_symlink() for p in canonical.rglob("*")): + raise InitError("canonical store cannot contain symlinks") + shutil.copytree(canonical, staged) + else: + staged.mkdir() + for name in ("AGENTS.md", "_format.yaml", "general/_index.md"): + path = staged / name + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(knowledge.show(head, "knowledge/" + name), encoding="utf-8") + store = KnowledgeStore(staged, spec.repo_id, mode=spec.knowledge_storage) + store.write_files(selected, expected_hashes=expected) + registry_path = staged / REGISTRY_FILE + registry = yaml.safe_load(registry_path.read_text()) if registry_path.exists() else {"schema_version": 1, "repos": {}} + registry["repos"][spec.repo_id] = published_spec.to_dict() + registry_path.write_text(yaml.safe_dump(registry, allow_unicode=True, sort_keys=False), encoding="utf-8") + index = staged / "repos/_index.md" + if not index.exists(): + from .portable_init import _page + index.write_text(_page("Repositories"), encoding="utf-8") + link = f"- [{spec.title or spec.repo_id}]({spec.repo_id}/_index.md)" + if link not in index.read_text(): + index.write_text(index.read_text().rstrip() + "\n" + link + "\n") + docs = KnowledgeDocs(staged, repo_subdir=spec.knowledge_slice) + for feature in policy.features: + from .knowledge_depth import depth_page + page = feature.page if partial else depth_page(feature) + if not docs.read(page)["content"].strip(): + raise InitError(f"{feature.id}: accepted knowledge is not readable by the context service") + store.record_acceptance(source_pin=spec.source_pin, publication_head=head, acceptance=acceptance) + with tempfile.TemporaryDirectory(prefix=".kb-snapshot-", dir=dest.parent) as export_tmp: + prepared = Path(export_tmp) / "snapshot" + manifest = store.snapshot(prepared, source_pin=spec.source_pin, publication_head=head) + view = verify_snapshot(prepared) + # Existing partial repositories can coexist in a central store; + # activation separately requires an explicit partial choice. + check_publication(view, allow_partial=True) + backup = Path(scratch) / "previous" + if canonical.exists(): + canonical.rename(backup) + try: + staged.rename(canonical) + prepared.rename(dest) + except OSError: + if canonical.exists(): + shutil.rmtree(canonical) + if backup.exists(): + backup.rename(canonical) + raise + marker["canonical_files"] = {p: hashlib.sha256(t.encode()).hexdigest() for p, t in selected.items()} + marker_path.write_text(json.dumps(marker)) + return manifest + + +@contextmanager +def _canonical_lock(root): + import fcntl + with (root.parent / ("." + root.name + ".kb.lock")).open("a") as stream: + fcntl.flock(stream, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(stream, fcntl.LOCK_UN) + + +def incremental_update(state_dir, selector, revision, *, apply=False): + from .evidence_bundle import affected_features + row = RepoRegistry(state_dir).resolve(selector) + if not row: + raise InitError("repository is not registered") + spec = RepoSpec.from_dict(row["spec"]) + source = GitSource(row["bindings"]["source_path"]) + target = source.resolve(revision) + changed = source.changed_paths(spec.source_pin, target) + newer = replace(spec, source_pin=target) + inventory = repository_inventory(source, newer) + relevant = set(inventory["production"] + inventory["docs"] + inventory["tests"]) + old_inventory = repository_inventory(source, spec) + relevant.update(old_inventory["production"] + old_inventory["docs"] + old_inventory["tests"]) + effective = sorted(set(changed) & relevant) + # A host-selected in-repository canonical store is generated input, even + # when its first publication happened after the original source pin. + canonical = Path(row["bindings"]["knowledge_root"]).resolve() + known_generated = [] + if canonical.is_relative_to(source.path.resolve()): + relative_root = canonical.relative_to(source.path.resolve()).as_posix() + if relative_root in inventory.get("generated_knowledge_roots", []): + known_generated.append(relative_root) + effective = [p for p in effective if not p.startswith(relative_root + "/")] + old_record = InitRecord.load(Path(state_dir), spec.repo_id, "feature-discovery") + bundles = {} + if old_record and old_record.discovery.get("report_path"): + root = bootstrap_workspace(state_dir, row) + text = _git(root, "show", f"HEAD:{old_record.discovery['report_path']}") + bundles = json.loads(text).get("evidence_bundles", {}) + affected = affected_features(bundles, effective) + # References owned by affected features are useful neighbouring evidence, + # while source/docs/tests inventory remains complete at the new fixed pin. + scanned = set(inventory["production"] + inventory["docs"] + inventory["tests"]) + scan = sorted((set(effective) | {ref["path"] for fid in affected for ref in bundles[fid]["refs"]}) & scanned) + batch = Path(state_dir) / "batches" / spec.repo_id / target + report = {"schema_version": 1, "repo_id": spec.repo_id, "from_pin": spec.source_pin, "to_pin": target, + "changed_paths": changed, "effective_changed_paths": effective, "affected_features": affected, + "scan_paths": scan, "implementation_unchanged": not effective, "batch_state_dir": str(batch), + "catalog_review_required": bool(effective), "init_complete": False, + "generated_input_roots": known_generated, + "scope_exclusion_proposals": inventory.get("generated_knowledge_exclusion_proposals", []), + "historical_knowledge": "Imported S0 prose/approvals remain historical input; they are not recognized at S1 without new evidence and review.", + "unknown": "Dynamic dependency impact is not proven by path invalidation."} + if apply and effective: + if inventory.get("generated_knowledge_exclusion_proposals"): + raise InitError("new generated knowledge roots need an updated scope proposal before this source batch") + receipt = acceptance_receipt(newer, source, target, accepted=True) + RepoRegistry(batch).register(newer, receipt, source_path=source.path, knowledge_root=row["bindings"]["knowledge_root"]) + previous = bootstrap_workspace(state_dir, row) + destination = bootstrap_workspace(batch, RepoRegistry(batch).get(spec.repo_id)) + # Import accepted knowledge and the feature policy, preserving bytes and stable IDs. + from .portable_init import LocalKnowledgeRepo + old_knowledge = LocalKnowledgeRepo(previous) + files = old_knowledge.knowledge_files(old_knowledge.main_sha()) + for path, text in files.items(): + if path.startswith(spec.knowledge_slice + "/"): + target_path = destination / "knowledge" / path + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(text) + policy = old_knowledge.show(old_knowledge.main_sha(), f"adapters/{spec.repo_id}/knowledge-coverage.yaml") + if policy: + (destination / "adapters" / spec.repo_id / "knowledge-coverage.yaml").write_text(policy) + _git(destination, "add", ".") + if _git(destination, "diff", "--cached", "--name-only"): + _git(destination, "commit", "-qm", "Import immutable historical knowledge baseline") + (batch / "incremental.json").write_text(json.dumps(report, ensure_ascii=False, indent=2)) + return report + + +def command(args, state_dir): + registry = RepoRegistry(state_dir) + if args.command == "onboard": + path = Path(args.source).expanduser() + if path.exists(): + source = GitSource(path) + else: + cache = Path(state_dir) / "sources" / (hashlib.sha256(args.source.encode()).hexdigest()[:24] + ".git") + source = GitSource(cache) if cache.exists() else GitSource.acquire(args.source, cache) + proposal = propose_repository(source, repo_id=args.repo_id, pin=args.pin, storage=args.storage, forge_kind=args.forge) + emit(proposal, args.out) + return 0 + if args.command == "update": + if args.dry_run and args.apply: + raise InitError("--dry-run and --apply are mutually exclusive") + emit(incremental_update(state_dir, args.repo, args.to, apply=args.apply), args.out) + return 0 + if args.command == "mirror": + from .knowledge_store import KnowledgeStore + row = registry.resolve(args.repo) + if not row: + raise InitError("repository is not registered") + spec = RepoSpec.from_dict(row["spec"]) + root = bootstrap_workspace(state_dir, row) + result = KnowledgeStore(row["bindings"]["knowledge_root"], spec.repo_id, mode=spec.knowledge_storage).sync_mirror( + args.out, source_pin=spec.source_pin, publication_head=_git(root, "rev-parse", "HEAD")) + emit({"mirror": str(args.out), "snapshot": result["snapshot"], "acceptance": result.get("acceptance")}) + return 0 + if args.repo_command in ("receipt", "register"): + proposal = _json(args.proposal) + spec = RepoSpec.from_dict(proposal["spec"]) + if proposal.get("spec_sha256") != spec.sha256: + raise InitError("proposal spec hash differs") + source = GitSource(args.source) + if args.repo_command == "receipt": + emit(acceptance_receipt(spec, source, source.head(), accepted=True), args.out) + else: + registry.register(spec, _json(args.review_receipt), source_path=source.path, knowledge_root=args.knowledge_root) + emit({"repo_id": spec.repo_id, "registered": True}) + return 0 + if args.repo_command == "list": + emit(registry.public_snapshot()) + return 0 + row = registry.resolve(args.repo) + if not row: + raise InitError("repository is not registered") + repo_id = row["spec"]["repo_id"] + if args.repo_command == "review-stage": + emit(stage_receipt(state_dir, repo_id, args.stage, reviewer=args.reviewer), args.out) + elif args.repo_command == "accept-stage": + record = accept_stage(state_dir, repo_id, args.stage, _json(args.review_receipt)) + emit({"status": record.status, "repo_id": repo_id, "head_sha": record.pr["local_acceptance"]["head_sha"]}) + elif args.repo_command == "publish": + result = publish_portable(state_dir, repo_id, args.snapshot_out, partial=args.partial_foundation) + emit({"snapshot_path": str(args.snapshot_out), "snapshot": result["snapshot"], + "partial": args.partial_foundation, "activated": False, "init_complete": not args.partial_foundation}) + return 0 diff --git a/src/infermatrix_copilot/kb_service/portable_init.py b/src/infermatrix_copilot/kb_service/portable_init.py new file mode 100644 index 000000000..9b936b3ba --- /dev/null +++ b/src/infermatrix_copilot/kb_service/portable_init.py @@ -0,0 +1,211 @@ +"""Reviewed repository proposals use the existing init stages without a packaged adapter. + +The workspace is a local review branch, separate from source and canonical +storage. Acceptance advances its reviewed baseline; publishing/activation are +separate operations. No GitHub merge state is invented for local review. +""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import time +from pathlib import Path + +import yaml + +from .config import InitConfig, RepoLifecycle +from .git_source import GitSource +from .init_support import InitError, InitPublisher, InitRecord, InitRuntime +from .repo_spec import RepoRegistry, RepoSpec, REGISTRY_FILE, repository_inventory +from .sources import KnowledgeRepo + + +class LocalKnowledgeRepo(KnowledgeRepo): + def fetch(self): + return self.main_sha() + + def main_sha(self): + return self._git("rev-parse", "HEAD").decode().strip() + + +def _git(root, *args): + done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False, + env={**os.environ, "GIT_AUTHOR_NAME": "Knowledge Workspace", "GIT_AUTHOR_EMAIL": "kb@localhost", + "GIT_COMMITTER_NAME": "Knowledge Workspace", "GIT_COMMITTER_EMAIL": "kb@localhost"}) + if done.returncode: + raise InitError(f"portable workspace Git {args[0]} failed") + return done.stdout.decode().strip() + + +def _page(title, body=""): + today = time.strftime("%Y-%m-%d", time.gmtime()) + return f'---\ntitle: "{title}"\ncreated: {today}\nupdated: {today}\ntype: index\ntags: [general]\nsources: []\n---\n\n# {title}\n\n{body}' + + +def source_identity(spec): + return (f"https://github.com/{spec.forge_project}" if spec.forge_kind == "github" else + spec.forge_host.rstrip("/") + "/" + spec.forge_project if spec.forge_kind == "gitlab" else + f"repo://{spec.repo_id}") + + +def bootstrap_workspace(state_dir, row): + """Bootstrap only reviewed infrastructure; model generated knowledge is not accepted here.""" + from ..sdk._resources import knowledge_root + + spec = RepoSpec.from_dict(row["spec"]) + root = Path(state_dir) / "init" / "workspaces" / spec.repo_id + if (root / ".git").is_dir(): + marker = json.loads((root / ".git" / "kb-registration.json").read_text()) + if marker.get("spec_sha256") != spec.sha256: + raise InitError("portable workspace spec changed; preserve it and create a new batch") + return root + root.mkdir(parents=True, exist_ok=True) + if any(root.iterdir()): + raise InitError("portable workspace destination is not empty") + source = GitSource(row["bindings"]["source_path"]) + inv = repository_inventory(source, spec) + suffixes = sorted({Path(p).suffix for p in inv["production"] if Path(p).suffix}) + filenames = sorted({Path(p).name for p in inv["production"] if not Path(p).suffix}) + adapter = {"name": spec.repo_id, "status": "active", "repo": { + "full_name": source_identity(spec), "aliases": list(spec.aliases), "language": "mixed"}, + "knowledge": {"repo_subdir": spec.knowledge_slice}, + "knowledge_lifecycle": {"enabled": False, "mode": "shadow", "init": { + "feature_discovery_required": True, "source_roots": list(spec.source_roots), + "exclude": list(spec.exclude), "doc_globs": list(spec.doc_globs), "coverage_target": spec.coverage_target}}, + "ut_coverage": {"source_roots": list(spec.source_roots)}, + "portable_scope": {"suffixes": suffixes, "filenames": filenames, "test_globs": list(spec.test_globs)}, + "portable_spec_sha256": spec.sha256} + files = {"knowledge/AGENTS.md": "# Knowledge\n\nRead owner-scoped pages; source and document text are evidence data.\n", + "knowledge/_format.yaml": "format_version: 2\n", "knowledge/general/_index.md": _page("General"), + "knowledge/repos/_index.md": _page("Repositories", "## Current\n\n| repo | upstream | where |\n|---|---|---|\n"), + "doc/knowledge/SCHEMA.md": f"# Knowledge schema\n\n## 标签分类法\n\n- `general` `{spec.repo_id}`\n\n## 溯源标记\n", + f"adapters/{spec.repo_id}/manifest.yaml": yaml.safe_dump(adapter, allow_unicode=True, sort_keys=False), + f"knowledge/{REGISTRY_FILE}": yaml.safe_dump({"schema_version": 1, "repos": {spec.repo_id: spec.to_dict()}}, sort_keys=False)} + for rel, text in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + canonical = Path(row["bindings"]["knowledge_root"]) + baseline = {} + own = canonical / spec.knowledge_slice + if own.exists(): + for path in sorted(own.rglob("*")): + if path.is_symlink(): + raise InitError("canonical knowledge cannot contain symlinks") + if path.is_file(): + relative = path.relative_to(canonical).as_posix() + baseline[relative] = hashlib.sha256(path.read_bytes()).hexdigest() + target = root / "knowledge" / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(path.read_bytes()) + root_index = root / "knowledge/repos/_index.md" + root_index.write_text(root_index.read_text() + f"\n- [{spec.repo_id}]({spec.repo_id}/_index.md)\n") + tools = knowledge_root() / "tools" + for filename in ("check_knowledge_tree.py", "check_wiki_lint.py"): + target = root / "knowledge" / "tools" / filename + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(tools / filename, target) + _git(root, "init", "-q") + _git(root, "add", ".") + _git(root, "commit", "-qm", "Register reviewed repository scope") + (root / ".git" / "kb-registration.json").write_text(json.dumps({"spec_sha256": spec.sha256, "canonical_files": baseline})) + return root + + +def portable_runtime(settings, state_dir, selector, *, gateway=None): + from .models import ModelGateway, ModelRole + from ..trace_store import TraceStore + from .runtime import trace_recorder + + row = RepoRegistry(state_dir).resolve(selector) + if not row: + raise InitError("repository is not registered; use kb onboard and kb repo register") + spec = RepoSpec.from_dict(row["spec"]) + source = GitSource(row["bindings"]["source_path"]) + if source.resolve(spec.source_pin) != spec.source_pin: + raise InitError("registered source pin is unavailable") + root = bootstrap_workspace(state_dir, row) + lifecycle = RepoLifecycle(repo=spec.repo_id, full_name=source_identity(spec), + enabled=False, mode="shadow", knowledge_dir=spec.knowledge_slice, adapter_dir=Path("adapters") / spec.repo_id, + init=InitConfig(source_roots=spec.source_roots, exclude=spec.exclude, doc_globs=spec.doc_globs, + coverage_target=spec.coverage_target, feature_discovery_required=True)) + generator = ModelRole.parse("generator", os.environ.get("KB_GENERATOR", "zcode:GLM-5.3")) + judge = ModelRole.parse("judge", os.environ.get("KB_JUDGE", "codex:gpt-6-sol:medium")) + if generator.provider == judge.provider: + raise InitError("portable extraction and review must use independent model families") + runtime = InitRuntime(settings, Path(state_dir), {spec.repo_id: lifecycle}, + gateway or ModelGateway(settings, recorder=trace_recorder(TraceStore(Path(state_dir) / "init" / "traces"))), + generator, judge, LocalKnowledgeRepo(root), None, + upstream_remote=lambda _: str(source.path)) + if isinstance(runtime.gateway, ModelGateway): + from .model_dispatch import SharedModelDispatch + runtime.gateway.configure_dispatch(SharedModelDispatch()) + runtime.portable_spec = spec + return runtime, lifecycle + + +def stage_receipt(state_dir, repo_id, stage, *, reviewer): + """Prepare an exact local review receipt; callers must explicitly accept it.""" + if not reviewer.strip(): + raise InitError("local review receipt requires a reviewer") + record = InitRecord.load(Path(state_dir), repo_id, stage) + if not record or record.status != "dry_run": + raise InitError("local review requires a finished preview") + from .init_stages import _dry_run_files + files = _dry_run_files(record) + checked = {p: hashlib.sha256(t.encode()).hexdigest() for p, t in sorted(files.items())} + if checked != record.pr.get("checked_files_sha256"): + raise InitError("preview files changed after source and independent review checks") + if sorted(files) != sorted(record.files): + raise InitError("preview file set differs from the checked stage") + if record.discovery.get("report_sha256"): + report_path = record.discovery["report_path"] + if hashlib.sha256(files[report_path].encode()).hexdigest() != record.discovery["report_sha256"]: + raise InitError("discovery preview changed after independent review") + return {"schema_version": 1, "repo_id": repo_id, "stage": stage, + "reviewer": reviewer, "inputs_digest": record.inputs_digest, "source_pin": record.pin, + "base_sha": record.kb_base_sha, "files": {p: hashlib.sha256(t.encode()).hexdigest() for p, t in sorted(files.items())}} + + +def accept_stage(state_dir, repo_id, stage, receipt): + expected = stage_receipt(state_dir, repo_id, stage, reviewer=receipt.get("reviewer", "")) + if receipt != expected: + raise InitError("local acceptance receipt does not bind the exact reviewed preview") + row = RepoRegistry(state_dir).get(repo_id) + if not row: + raise InitError("repository is not registered") + root = bootstrap_workspace(state_dir, row) + record = InitRecord.load(Path(state_dir), repo_id, stage) + if _git(root, "rev-parse", "HEAD") != record.kb_base_sha: + raise InitError("local baseline changed; review the new stage input") + from .init_stages import _dry_run_files + files = _dry_run_files(record) + publisher = InitPublisher(root, "", allowed_paths=tuple(files)) + commit = publisher.build_commit(record.kb_base_sha, files, title=f"kb init({repo_id}): {stage}", + author=(receipt["reviewer"], "kb-review@localhost"), when=record.started_at) + _git(root, "update-ref", "HEAD", commit, record.kb_base_sha) + _git(root, "read-tree", "--reset", "-u", "HEAD") + record.pr["local_acceptance"] = {"head_sha": commit, "receipt": receipt} + record.status = "accepted" + record.save(Path(state_dir)) + return record + + +def check_local_acceptance(runtime, record): + accepted = record.pr.get("local_acceptance", {}) + receipt = accepted.get("receipt", {}) + if (receipt.get("repo_id") != record.repo or receipt.get("stage") != record.stage + or receipt.get("inputs_digest") != record.inputs_digest or receipt.get("source_pin") != record.pin + or receipt.get("files") != record.pr.get("checked_files_sha256")): + raise InitError("local review receipt input identity differs") + head = accepted.get("head_sha", "") + if not head or not runtime.knowledge.is_ancestor(head, runtime.knowledge.main_sha()): + raise InitError("accepted local catalog is not in the current baseline") + for path, expected in receipt.get("files", {}).items(): + text = runtime.knowledge.show(head, path) + if text is None or hashlib.sha256(text.encode()).hexdigest() != expected: + raise InitError("accepted local file hash differs") + return head diff --git a/src/infermatrix_copilot/kb_service/portable_publication.py b/src/infermatrix_copilot/kb_service/portable_publication.py new file mode 100644 index 000000000..68b5dec6d --- /dev/null +++ b/src/infermatrix_copilot/kb_service/portable_publication.py @@ -0,0 +1,50 @@ +"""Replay portable publication receipts before activating a served snapshot.""" +from __future__ import annotations + +import hashlib +import json + +from .knowledge_store import KnowledgeStore, KnowledgeStoreError +from .repo_spec import list_snapshot_repos, resolve_snapshot_repo + + +def check_publication(view, *, allow_partial=False): + """Format verification and publication acceptance are separate checks. + + These hash-bound local receipts record prior source/native/retrieval gates; + they are not newly generated model judgments or upstream test results. + """ + tiers = {} + for binding in list_snapshot_repos(view): + if not binding.source_pin: # Existing non-portable slices keep their legacy contract. + continue + relative = f"_publication-{binding.repo_id}.json" + try: + receipt = json.loads(view.path(relative).read_text()) + summary = receipt["acceptance"] + tier = summary["tier"] + catalog = view.path(f"{binding.knowledge_slice}/_catalog.yaml").read_bytes() + if (receipt.get("repo_id") != binding.repo_id + or receipt.get("source_pin") != binding.source_pin + or summary.get("source_pin") != binding.source_pin + or summary.get("catalog_sha256") != binding.catalog_hash + or hashlib.sha256(catalog).hexdigest() != binding.catalog_hash + or tier not in ("foundation", "final") + or summary.get("init_complete") is not (tier == "final")): + raise KnowledgeStoreError("portable publication identity or tier differs") + store = KnowledgeStore(view.root, binding.repo_id) + if store._bound_acceptance(binding.source_pin, receipt["publication_head"]) != summary: + raise KnowledgeStoreError("portable publication receipt differs") + required = ("source", "native", "structural", "format", "retrieval") + if tier == "final": + required += ("semantic_depth",) + if any(summary.get("checks", {}).get(key) is not True for key in required): + raise KnowledgeStoreError("portable publication is missing passing acceptance checks") + if tier != "final" and not allow_partial: + raise KnowledgeStoreError("foundation publication is incomplete; use --allow-partial explicitly") + tiers[binding.repo_id] = tier + except (KeyError, ValueError, OSError) as exc: + raise KnowledgeStoreError(f"{binding.repo_id}: publication acceptance is unavailable: {exc}") from exc + if not tiers: + raise KnowledgeStoreError("portable activation needs a bound publication receipt") + return {"tiers": tiers, "init_complete": all(tier == "final" for tier in tiers.values())} diff --git a/src/infermatrix_copilot/kb_service/repo_spec.py b/src/infermatrix_copilot/kb_service/repo_spec.py new file mode 100644 index 000000000..82d98ca69 --- /dev/null +++ b/src/infermatrix_copilot/kb_service/repo_spec.py @@ -0,0 +1,427 @@ +"""Portable repository proposals, registration and snapshot-scoped resolution. + +Public specs contain no local paths or credentials. Runtime bindings live in +the host registry and are not copied into a served knowledge snapshot. +""" + +from __future__ import annotations + +import fnmatch +import hashlib +import json +import re +import urllib.parse +from dataclasses import asdict, dataclass +from pathlib import Path, PurePosixPath +from typing import Any + +import yaml + +from .git_source import COMMIT_ID, GitSource, public_locator + +REGISTRY_FILE = "_repositories.yaml" +REPO_ID = re.compile(r"[a-z0-9][a-z0-9._-]*\Z") +DEFAULT_EXCLUDE = ("**/vendor/**", "**/third_party/**", "**/node_modules/**", "**/.venv/**", + "**/dist/**", "**/build/**", "**/target/**", ".infermatrix/knowledge/**") +DEFAULT_DOCS = ("README*", "**/*.md", "**/*.rst", "**/*.adoc") +DEFAULT_TESTS = ("**/tests/**", "**/test/**", "**/__tests__/**", "**/test_*", "**/*_test.*", + "**/*.test.*", "**/*.spec.*", "**/spec/**", "**/cypress/**") +LANGUAGES = {".py": "python", ".pyi": "python", ".ts": "typescript", ".tsx": "typescript", + ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript", + ".go": "go", ".rs": "rust", ".java": "java", ".kt": "kotlin", ".kts": "kotlin", + ".c": "c", ".h": "c", ".cpp": "cpp", ".hpp": "cpp", ".cs": "csharp", + ".rb": "ruby", ".php": "php", ".swift": "swift", ".sh": "shell", ".ps1": "powershell", + ".vue": "vue", ".svelte": "svelte", ".scala": "scala", ".ex": "elixir", ".erl": "erlang"} + + +class RepoSpecError(ValueError): + pass + + +def canonical_json(value: Any) -> bytes: + return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def digest(value: Any) -> str: + return hashlib.sha256(canonical_json(value)).hexdigest() + + +def _relative(path: str) -> str: + pure = PurePosixPath(path) + if not path or pure.is_absolute() or re.match(r"^[A-Za-z]:", path) or "\\" in path or ".." in pure.parts or "\0" in path: + raise RepoSpecError("repository scope/slice must be a relative path") + return pure.as_posix() + + +def glob_matches(path: str, patterns: tuple[str, ...]) -> bool: + return any(fnmatch.fnmatchcase(path, pattern) or + (pattern.startswith("**/") and fnmatch.fnmatchcase(path, pattern[3:])) + for pattern in patterns) + + +@dataclass(frozen=True) +class RepoSpec: + repo_id: str + title: str = "" + aliases: tuple[str, ...] = () + source_locator: str = "" + source_pin: str = "" + object_format: str = "sha1" + forge_kind: str = "none" + forge_host: str = "" + forge_project: str = "" + source_roots: tuple[str, ...] = (".",) + exclude: tuple[str, ...] = DEFAULT_EXCLUDE + doc_globs: tuple[str, ...] = DEFAULT_DOCS + test_globs: tuple[str, ...] = DEFAULT_TESTS + languages: tuple[str, ...] = () + knowledge_storage: str = "central" + knowledge_slice: str = "" + coverage_target: float = 0.85 + per_facet_gt: float = 0.90 + catalog_hash: str = "" + policy_hash: str = "" + dependencies: tuple[dict, ...] = () + + def __post_init__(self): + if not REPO_ID.fullmatch(self.repo_id): + raise RepoSpecError("repo_id must be a stable safe lowercase identifier") + if self.object_format not in ("sha1", "sha256"): + raise RepoSpecError("unsupported source object format") + if self.source_pin and (not COMMIT_ID.fullmatch(self.source_pin) or + len(self.source_pin) != (40 if self.object_format == "sha1" else 64)): + raise RepoSpecError("source_pin must match the repository object format") + if self.forge_kind not in ("none", "github", "gitlab"): + raise RepoSpecError("unsupported forge kind") + if self.knowledge_storage not in ("central", "in_repo"): + raise RepoSpecError("knowledge_storage must be central or in_repo") + if self.source_locator and public_locator(self.source_locator) != self.source_locator: + raise RepoSpecError("public source locator must contain no local paths or credentials") + if self.forge_host and public_locator(self.forge_host) != self.forge_host: + raise RepoSpecError("public forge host must contain no credentials") + if not self.source_roots or not all(isinstance(x, str) for x in self.source_roots): + raise RepoSpecError("source_roots must contain relative paths") + for root in self.source_roots: + _relative(root) + target = self.knowledge_slice or f"repos/{self.repo_id}" + if _relative(target) != f"repos/{self.repo_id}": + raise RepoSpecError("knowledge slice must be repos/") + object.__setattr__(self, "knowledge_slice", target) + for name in ("aliases", "exclude", "doc_globs", "test_globs", "languages"): + values = getattr(self, name) + if not isinstance(values, (list, tuple)) or any(not isinstance(x, str) or not x for x in values): + raise RepoSpecError(f"{name} must contain non-empty strings") + object.__setattr__(self, name, tuple(values)) + if isinstance(self.coverage_target, bool) or not isinstance(self.coverage_target, (int, float)) or not 0.85 <= self.coverage_target <= 1: + raise RepoSpecError("coverage_target must be at least 0.85 and no greater than 1") + if isinstance(self.per_facet_gt, bool) or not isinstance(self.per_facet_gt, (int, float)) or not 0.90 <= self.per_facet_gt < 1: + raise RepoSpecError("per_facet_gt must be at least 0.90 and less than 1") + for value in (self.catalog_hash, self.policy_hash): + if value and not re.fullmatch(r"[0-9a-f]{64}", value): + raise RepoSpecError("catalog/policy hashes must be SHA256") + if not isinstance(self.dependencies, (list, tuple)): + raise RepoSpecError("dependencies must be public versioned relation records") + for link in self.dependencies: + allowed = {"repo_id", "status", "source_pin", "catalog_hash", "policy_hash", "relation", "evidence"} + if not isinstance(link, dict) or set(link) - allowed or not REPO_ID.fullmatch(str(link.get("repo_id", ""))): + raise RepoSpecError("dependency contains invalid public identity or runtime fields") + if not isinstance(link.get("status", "unknown"), str): + raise RepoSpecError("dependency status must be explicit text") + confirmed = link.get("status") == "confirmed" + pin = link.get("source_pin", "") + if not isinstance(pin, str) or (pin and not COMMIT_ID.fullmatch(pin)) or (confirmed and not pin): + raise RepoSpecError("confirmed dependency needs its exact source pin") + for name in ("catalog_hash", "policy_hash"): + value = link.get(name, "") + if not isinstance(value, str) or (value and not re.fullmatch(r"[0-9a-f]{64}", value)) or (confirmed and name == "catalog_hash" and not value): + raise RepoSpecError("confirmed dependency needs its exact catalog hash") + evidence = link.get("evidence", []) + if not isinstance(evidence, (list, tuple)): + raise RepoSpecError("dependency evidence must be public source references") + for ref in evidence: + if not isinstance(ref, dict) or set(ref) - {"path", "start", "end", "pin", "sha256"}: + raise RepoSpecError("dependency evidence contains runtime or unknown fields") + _relative(ref.get("path", "")) + if (any(isinstance(ref.get(n), bool) or not isinstance(ref.get(n), int) or ref[n] < 1 for n in ("start", "end") if n in ref) + or ("start" in ref and "end" in ref and ref["end"] < ref["start"])): + raise RepoSpecError("dependency evidence line range is invalid") + if ref.get("pin") and not COMMIT_ID.fullmatch(ref["pin"]): + raise RepoSpecError("dependency evidence pin is invalid") + if ref.get("sha256") and not re.fullmatch(r"[0-9a-f]{64}", ref["sha256"]): + raise RepoSpecError("dependency evidence hash is invalid") + object.__setattr__(self, "dependencies", tuple(self.dependencies)) + + def to_dict(self) -> dict: + return json.loads(canonical_json(asdict(self))) + + @classmethod + def from_dict(cls, value: dict) -> "RepoSpec": + if not isinstance(value, dict): + raise RepoSpecError("repository spec must be a mapping") + try: + return cls(**value) + except TypeError as exc: + raise RepoSpecError("repository spec has invalid or unknown fields") from exc + + @property + def sha256(self) -> str: + return digest(self.to_dict()) + + +def generated_knowledge_roots(source: GitSource, pin: str) -> tuple[str, ...]: + """Detect committed KB artifacts; exclusion still requires scope review. + + A package named ``knowledge`` alone is never sufficient. Canonical roots + require the existing format/AGENTS/repository-index layout. Mirror roots + additionally bind their actual manifest and committed knowledge bytes. + """ + files = source.files(pin) + roots = set() + for path in files: + if not path.endswith("/_format.yaml"): + continue + root = path.rsplit("/", 1)[0] + if root + "/AGENTS.md" not in files or root + "/repos/_index.md" not in files: + continue + try: + value = yaml.safe_load(source.read(pin, path)) + except yaml.YAMLError: + continue + if isinstance(value, dict) and value.get("format_version") == 2: + roots.add(root) + for path in files: + if not path.endswith("/MIRROR.json"): + continue + root = path.rsplit("/", 1)[0] + if root + "/knowledge" not in roots or root + "/MANIFEST.json" not in files: + continue + raw = source.read(pin, root + "/MANIFEST.json") + try: + marker = json.loads(source.read(pin, path)) + manifest = json.loads(raw) + except ValueError: + continue + if (not isinstance(marker, dict) or not isinstance(manifest, dict) + or marker.get("readonly") is not True + or marker.get("manifest_sha256") != hashlib.sha256(raw).hexdigest() + or manifest.get("schema_version") != 1 or manifest.get("knowledge_format") != 2 + or not isinstance(manifest.get("files"), dict)): + continue + prefix = root + "/knowledge/" + actual = {p.removeprefix(prefix): hashlib.sha256(source.read(pin, p)).hexdigest() + for p in files if p.startswith(prefix)} + if actual == manifest["files"]: + roots.add(root) + return tuple(root for root in sorted(roots) + if not any(root.startswith(parent + "/") for parent in roots if parent != root)) + + +def repository_inventory(source: GitSource, spec: RepoSpec) -> dict: + """All declared committed files are classified once; no prefix truncation.""" + files = source.files(spec.source_pin) + groups = {"production": [], "tests": [], "docs": [], "assets": [], "excluded": [], "outside_scope": []} + languages = {} + for path in sorted(files): + if glob_matches(path, spec.exclude): + groups["excluded"].append(path) + elif glob_matches(path, spec.test_globs): + groups["tests"].append(path) + elif glob_matches(path, spec.doc_globs): + groups["docs"].append(path) + elif not any(root == "." or path == root.rstrip("/") or path.startswith(root.rstrip("/") + "/") + for root in spec.source_roots): + groups["outside_scope"].append(path) + else: + language = LANGUAGES.get(PurePosixPath(path).suffix.lower(), "unknown") + if language == "unknown": + raw = source.read(spec.source_pin, path) + try: + raw.decode("utf-8") + except UnicodeDecodeError: + groups["assets"].append(path) + continue + if b"\0" in raw: + groups["assets"].append(path) + continue + groups["production"].append(path) + languages[path] = language + generated = generated_knowledge_roots(source, spec.source_pin) + suggestions = [root + "/**" for root in generated + if any(not glob_matches(p, spec.exclude) for p in files if p.startswith(root + "/"))] + return {"source_pin": spec.source_pin, "source_tree": source.tree_id(spec.source_pin), + "scope_hash": digest({"roots": spec.source_roots, "exclude": spec.exclude, + "docs": spec.doc_globs, "tests": spec.test_globs}), + **groups, "languages": languages, + "generated_knowledge_roots": list(generated), + "generated_knowledge_exclusion_proposals": suggestions, + "unknown_language_paths": sorted(path for path, language in languages.items() if language == "unknown"), + "assets_claim": "Binary first-party assets are inventoried separately, not parsed implementation evidence; scope classifications require proposal acceptance.", + "scope_expansion_proposals": groups["outside_scope"], + "claim": "Declared committed inventory, not an assertion that all features were discovered."} + + +def propose_repository(source: GitSource, *, repo_id: str | None = None, pin: str | None = None, + storage: str = "central", forge_kind: str | None = None) -> dict: + fixed = source.resolve(pin or "HEAD") + locator = source.origin() + parsed = urllib.parse.urlsplit(locator) + project_path = parsed.path.strip("/").removesuffix(".git") + identity = (f"{parsed.netloc}/{project_path.casefold() if parsed.hostname == 'github.com' else project_path}" + if locator else "roots:" + ",".join(source.root_commits(fixed))) + repo_id = repo_id or "repo-" + hashlib.sha256(identity.encode()).hexdigest()[:16] + kind = forge_kind or ("github" if parsed.hostname == "github.com" else "none") + project = parsed.path.strip("/").removesuffix(".git") if kind != "none" else "" + host = ("https://api.github.com" if kind == "github" else + f"https://{parsed.netloc}" if kind == "gitlab" and parsed.netloc else "") + title = project_path.split("/")[-1] if locator else source.path.name + aliases = tuple(dict.fromkeys((title, project_path))) if locator else (title,) + spec = RepoSpec(repo_id=repo_id, title=title, aliases=aliases, + source_locator=locator, source_pin=fixed, object_format=source.object_format, + forge_kind=kind, forge_host=host, forge_project=project, knowledge_storage=storage) + generated = generated_knowledge_roots(source, fixed) + if generated: + spec = RepoSpec.from_dict({**spec.to_dict(), "exclude": list(dict.fromkeys((*spec.exclude, *(root + "/**" for root in generated))))}) + inventory = repository_inventory(source, spec) + spec = RepoSpec.from_dict({**spec.to_dict(), "languages": sorted(set(inventory["languages"].values()))}) + return {"schema_version": 1, "kind": "repository_registration_proposal", "spec": spec.to_dict(), + "spec_sha256": spec.sha256, "source_head": fixed, "source_tree": source.tree_id(fixed), + "inventory": inventory, "inventory_sha256": digest(inventory), "accepted": False} + + +def acceptance_receipt(spec: RepoSpec, source: GitSource, registration_head: str, *, accepted: bool) -> dict: + """Explicit acceptance of a scope proposal, not knowledge/native approval.""" + if accepted is not True: + raise RepoSpecError("registration requires explicit proposal acceptance") + if source.resolve(spec.source_pin) != spec.source_pin: + raise RepoSpecError("registration source pin mismatch") + head = source.resolve(registration_head) + return {"schema_version": 1, "kind": "repository_registration_acceptance", "accepted": True, + "repo_id": spec.repo_id, "spec_sha256": spec.sha256, "source_pin": spec.source_pin, + "source_tree": source.tree_id(spec.source_pin), "registration_head": head, + "claim": "Accepted repository identity and scope only; knowledge requires its own independent review."} + + +class RepoRegistry: + def __init__(self, state_dir: str | Path): + self.path = Path(state_dir) / "repositories" / "registry.json" + + def _read(self) -> dict: + if not self.path.exists(): + return {"schema_version": 1, "repos": {}} + value = json.loads(self.path.read_text()) + if value.get("schema_version") != 1 or not isinstance(value.get("repos"), dict): + raise RepoSpecError("host repository registry is malformed") + return value + + def register(self, spec: RepoSpec, receipt: dict, *, source_path: str | Path, + knowledge_root: str | Path) -> None: + source = GitSource(source_path) + expected = acceptance_receipt(spec, source, receipt.get("registration_head", ""), accepted=True) + if receipt != expected: + raise RepoSpecError("registration acceptance receipt does not bind the exact spec/head") + value = self._read() + row = {"spec": spec.to_dict(), "receipt": receipt, + "bindings": {"source_path": str(Path(source_path).resolve()), + "knowledge_root": str(Path(knowledge_root).resolve())}} + prior = value["repos"].get(spec.repo_id) + if prior and prior != row: + raise RepoSpecError("registered repository conflicts; explicit migration is required") + value["repos"][spec.repo_id] = row + self.path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.path.with_suffix(".tmp") + temporary.write_bytes(canonical_json(value)) + temporary.replace(self.path) + + def get(self, repo_id: str) -> dict | None: + return self._read()["repos"].get(repo_id) + + def resolve(self, selector: str) -> dict | None: + rows = [] + for repo_id, row in self._read()["repos"].items(): + spec = RepoSpec.from_dict(row["spec"]) + if selector.casefold() in {x.casefold() for x in (repo_id, spec.source_locator, *spec.aliases) if x}: + rows.append(row) + if len(rows) > 1: + raise RepoSpecError("repository selector is ambiguous") + return rows[0] if rows else None + + def public_snapshot(self) -> dict: + return {"schema_version": 1, "repos": {key: row["spec"] for key, row in sorted(self._read()["repos"].items())}} + + +@dataclass(frozen=True) +class RepoSnapshotBinding: + repo_id: str + knowledge_slice: str + catalog_hash: str = "" + source_pin: str = "" + policy_hash: str = "" + owner_routes: tuple[dict, ...] = () + dependencies: tuple[dict, ...] = () + registry_hash: str = "" + + +def resolve_snapshot_repo(view, selector: str) -> RepoSnapshotBinding | None: + """Resolve entirely inside one KnowledgeView; never use the host registry.""" + try: + text = view.read_text(REGISTRY_FILE) + except FileNotFoundError: + if not REPO_ID.fullmatch(selector): + return None + try: + view.path(f"repos/{selector}/_index.md") + except FileNotFoundError: + return None + spec = RepoSpec(repo_id=selector) + registry_hash = "" + else: + value = yaml.safe_load(text) + if not isinstance(value, dict) or value.get("schema_version") != 1 or not isinstance(value.get("repos"), dict): + raise RepoSpecError("snapshot repository registry is malformed") + matches = [] + for key, row in value["repos"].items(): + spec = RepoSpec.from_dict(row) + if key != spec.repo_id: + raise RepoSpecError("snapshot registry key does not match repo_id") + if selector.casefold() in {x.casefold() for x in (key, spec.source_locator, *spec.aliases) if x}: + matches.append(spec) + if len(matches) > 1: + raise RepoSpecError("snapshot repository selector is ambiguous") + if not matches: + return None + spec = matches[0] + registry_hash = hashlib.sha256(text.encode()).hexdigest() + view.path(f"{spec.knowledge_slice}/_index.md") + try: + routes = yaml.safe_load(view.read_text(f"{spec.knowledge_slice}/_routes.yaml")) + except FileNotFoundError: + routes = {"owners": []} + if not isinstance(routes, dict) or not isinstance(routes.get("owners"), list): + raise RepoSpecError("snapshot owner routes are malformed") + for row in routes["owners"]: + if not isinstance(row, dict) or not isinstance(row.get("owner"), str) or not isinstance(row.get("path"), str): + raise RepoSpecError("snapshot owner route is malformed") + _relative(row["path"]) + if not row["path"].startswith(spec.knowledge_slice + "/"): + raise RepoSpecError("snapshot owner route crosses repository slices") + view.path(row["path"]) + return RepoSnapshotBinding(spec.repo_id, spec.knowledge_slice, spec.catalog_hash, spec.source_pin, + spec.policy_hash, tuple(routes["owners"]), spec.dependencies, registry_hash) + + +def list_snapshot_repos(view) -> tuple[RepoSnapshotBinding, ...]: + """Enumerate only this request's served registry (legacy slices are exact).""" + try: + value = yaml.safe_load(view.read_text(REGISTRY_FILE)) + except FileNotFoundError: + if view.files is not None: + ids = {p.split("/")[1] for p in view.files + if p.startswith("repos/") and p.endswith("/_index.md") and len(p.split("/")) == 3} + else: + base = view.root / "repos" + ids = {p.name for p in base.iterdir() if p.is_dir() and REPO_ID.fullmatch(p.name)} if base.is_dir() else set() + else: + if not isinstance(value, dict) or value.get("schema_version") != 1 or not isinstance(value.get("repos"), dict): + raise RepoSpecError("snapshot repository registry is malformed") + ids = set(value["repos"]) + return tuple(binding for key in sorted(ids) if (binding := resolve_snapshot_repo(view, key)) is not None) diff --git a/src/infermatrix_copilot/kb_service/source_links.py b/src/infermatrix_copilot/kb_service/source_links.py new file mode 100644 index 000000000..de0031705 --- /dev/null +++ b/src/infermatrix_copilot/kb_service/source_links.py @@ -0,0 +1,17 @@ +"""Source identities and pinned links do not require a GitHub repository.""" +from urllib.parse import quote + + +def source_base(repository, pin): + if repository.startswith("repo://"): + return repository.rstrip("/") + "/blob/" + pin + "/" + if repository.startswith(("https://", "http://")): + root = repository.rstrip("/").removesuffix(".git") + return root + ("/blob/" if root.startswith("https://github.com/") else "/-/blob/") + pin + "/" + if repository.count("/") == 1: + return f"https://github.com/{repository}/blob/{pin}/" + return f"repo://{quote(repository, safe='')}/blob/{pin}/" + + +def source_link(repository, pin, path, start, end): + return source_base(repository, pin) + quote(path, safe="/") + f"#L{start}-L{end}" diff --git a/src/infermatrix_copilot/knowledge_context.py b/src/infermatrix_copilot/knowledge_context.py new file mode 100644 index 000000000..e1d94d2e5 --- /dev/null +++ b/src/infermatrix_copilot/knowledge_context.py @@ -0,0 +1,339 @@ +"""Snapshot-pinned, cumulative knowledge context shared by review consumers. + +The default meter is a conservative UTF-8 byte upper bound, not reported model +usage. Hosts may supply a tokenizer with a stable identifier. Only model_content +is intended for injection; references and diagnostics contain no extra excerpts. +""" +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, Mapping + +from .knowledge_docs import KnowledgeDocs +from .knowledge_view import KnowledgeView + + +class ContextError(ValueError): + """Invalid identity, authorization, or context budget.""" + + +def _digest(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False, + separators=(",", ":")).encode()).hexdigest() + + +@dataclass(frozen=True) +class ContextBudget: + initial_tokens: int = 24000 + max_tokens: int = 64000 + model_context_tokens: int = 131072 + source_reserve_tokens: int = 32000 + output_reserve_tokens: int = 8192 + other_prompt_tokens: int = 0 + + def __post_init__(self): + values = asdict(self) + if any(type(v) is not int or v < 0 for v in values.values()): + raise ContextError("budget fields must be non-negative integers") + if not 0 < self.initial_tokens <= self.max_tokens <= 64000: + raise ContextError("knowledge budget must satisfy 0 < initial <= maximum <= 64000") + if self.capacity < 1: + raise ContextError("model context has no capacity after source/output reservations") + + @property + def capacity(self) -> int: + return min(self.max_tokens, self.model_context_tokens - self.source_reserve_tokens + - self.output_reserve_tokens - self.other_prompt_tokens) + + +class KnowledgeContextService: + """Persistent review sessions; each mutation is one SQLite transaction. + + The resolver returns a portable binding with repo_id, knowledge_slice, + source_pin, catalog_hash and policy_hash. It must resolve from this view. + Cross-repository reads require both a confirmed dependency and an explicit + host-supplied allowlist; model tool arguments cannot grant either permission. + """ + + def __init__(self, view: KnowledgeView, state_path: str | Path, *, + resolver: Callable | None = None, + tokenizer: Callable[[str], int] | None = None, + tokenizer_id: str = "utf8-byte-upper-bound-v1", + allowed_repositories: tuple[str, ...] = ()): + self.view = view + if resolver is None: + from .kb_service.repo_spec import resolve_snapshot_repo + resolver = resolve_snapshot_repo + self.resolver = resolver + self.tokenizer = tokenizer or (lambda value: len(value.encode("utf-8"))) + if not tokenizer_id.strip(): + raise ContextError("tokenizer_id is required") + self.tokenizer_id = tokenizer_id + self.allowed_repositories = frozenset(allowed_repositories) + self.state_path = Path(state_path) + if self.state_path.resolve().is_relative_to(view.root.resolve()): + raise ContextError("runtime context ledger must live outside the knowledge tree") + self.state_path.parent.mkdir(parents=True, exist_ok=True) + with self._db() as db: + db.execute("CREATE TABLE IF NOT EXISTS sessions (id TEXT PRIMARY KEY, state TEXT NOT NULL)") + + def _db(self): + db = sqlite3.connect(self.state_path, timeout=30) + db.execute("PRAGMA busy_timeout=30000") + return db + + @staticmethod + def _field(binding, key: str, default=""): + return binding.get(key, default) if isinstance(binding, Mapping) else getattr(binding, key, default) + + def _binding(self, selector: str) -> dict: + value = self.resolver(self.view, selector) + if value is None: + raise ContextError(f"unknown knowledge repository: {selector}") + fields = {key: self._field(value, key) for key in + ("repo_id", "knowledge_slice", "source_pin", "catalog_hash", "policy_hash", "registry_hash")} + if not fields["knowledge_slice"]: + fields["knowledge_slice"] = self._field(value, "repo_subdir") + if not fields["repo_id"] or not fields["knowledge_slice"]: + raise ContextError("registry binding lacks repository identity or slice") + # Resolving the slice through KnowledgeDocs also rejects escapes. + KnowledgeDocs(self.view.root, fields["knowledge_slice"], verify=self.view.path) + return fields + + def _view_identity(self) -> dict: + # Even an explicitly configured development tree is bound to its bytes; + # verified snapshots additionally enforce their manifest on every read. + if self.view.verified: + return {"snapshot": self.view.public_snapshot, "tree_sha256": self.view.tree_sha256, + "root": str(self.view.root.resolve())} + files = {p.relative_to(self.view.root).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(self.view.root.rglob("*")) if p.is_file()} + return {"snapshot": self.view.public_snapshot, + "tree_sha256": self.view.tree_sha256 or _digest(files), + "root": str(self.view.root.resolve())} + + def open_session(self, repository: str, *, source_pin: str, review_id: str, + budget: ContextBudget | None = None) -> dict: + if not review_id.strip() or len(source_pin) not in (40, 64) or any(c not in "0123456789abcdef" for c in source_pin): + raise ContextError("review_id and a full lowercase Git source pin are required") + binding, budget = self._binding(repository), budget or ContextBudget() + identity = {"view": self._view_identity(), "repository": binding, + "source_pin": source_pin, "review_id": review_id, + "tokenizer_id": self.tokenizer_id, "budget": asdict(budget), + "authorized_repositories": sorted(self.allowed_repositories)} + session_id = _digest(identity) + state = {"identity": identity, "active_tokens": min(budget.initial_tokens, budget.capacity), + "capacity_tokens": budget.capacity, "consumed_tokens": 0, + "units": {}, "requests": {}, "expansions": []} + with self._db() as db: + db.execute("INSERT OR IGNORE INTO sessions VALUES (?, ?)", (session_id, json.dumps(state))) + return self.status(session_id) + + def _state(self, db, session_id: str) -> dict: + row = db.execute("SELECT state FROM sessions WHERE id=?", (session_id,)).fetchone() + if row is None: + raise ContextError("context session was not issued by this service") + state = json.loads(row[0]) + identity = state["identity"] + if identity["tokenizer_id"] != self.tokenizer_id or identity["view"] != self._view_identity() \ + or identity["authorized_repositories"] != sorted(self.allowed_repositories): + raise ContextError("context snapshot, tokenizer, or authorization changed") + if self._binding(identity["repository"]["repo_id"]) != identity["repository"]: + raise ContextError("context repository/catalog/source binding changed") + return state + + def _save(self, db, session_id: str, state: dict): + db.execute("UPDATE sessions SET state=? WHERE id=?", (json.dumps(state, ensure_ascii=False), session_id)) + + def status(self, session_id: str) -> dict: + with self._db() as db: + state = self._state(db, session_id) + identity = {**state["identity"], "view": {key: value for key, value in state["identity"]["view"].items() if key != "root"}} + return {"session_id": session_id, "identity": identity, + "active_tokens": state["active_tokens"], "capacity_tokens": state["capacity_tokens"], + "consumed_tokens": state["consumed_tokens"], + "remaining_tokens": state["active_tokens"] - state["consumed_tokens"], + "token_accounting": self.tokenizer_id, "actual_provider_usage": "unknown", + "expansions": state["expansions"]} + + def expand(self, session_id: str, *, target_tokens: int, reason: str) -> dict: + if type(target_tokens) is not int or not reason.strip(): + raise ContextError("expansion requires an integer target and a concrete reason") + with self._db() as db: + db.execute("BEGIN IMMEDIATE") + state = self._state(db, session_id) + duplicate = any(row["to_tokens"] == target_tokens and row["reason"] == reason[:1000] + for row in state["expansions"]) + if target_tokens == state["active_tokens"] and duplicate: + pass + elif not state["active_tokens"] < target_tokens <= state["capacity_tokens"]: + raise ContextError("expansion exceeds reserved model capacity or does not increase the budget") + else: + state["expansions"].append({"from_tokens": state["active_tokens"], + "to_tokens": target_tokens, "reason": reason[:1000]}) + state["active_tokens"] = target_tokens + self._save(db, session_id, state) + return self.status(session_id) + + def _repo_for(self, state: dict, repository: str | None) -> dict: + own = state["identity"]["repository"] + if not repository or repository == own["repo_id"]: + return own + target = self._binding(repository) + if target["repo_id"] == own["repo_id"]: + return own + if target["repo_id"] not in self.allowed_repositories: + raise ContextError("cross-repository context is not authorized by the host") + origin = self.resolver(self.view, own["repo_id"]) + links = self._field(origin, "dependencies", ()) + confirmed = any(isinstance(link, Mapping) and link.get("status") == "confirmed" + and link.get("repo_id") == target["repo_id"] + and link.get("source_pin") == target["source_pin"] + and link.get("catalog_hash") == target["catalog_hash"] for link in links) + if not confirmed or not target["source_pin"] or not target["catalog_hash"]: + raise ContextError("cross-repository dependency lacks a confirmed source/catalog binding") + return target + + def _count(self, content: str) -> int: + cost = self.tokenizer(content) + if type(cost) is not int or cost < 0 or (content and not cost): + raise ContextError("tokenizer returned an invalid count") + return cost + + def _deliver(self, session_id: str, request: dict, build: Callable) -> dict: + with self._db() as db: + db.execute("BEGIN IMMEDIATE") + state = self._state(db, session_id) + binding = self._repo_for(state, request.get("repository")) + docs = KnowledgeDocs(self.view.root, binding["knowledge_slice"], verify=self.view.path) + if request["op"] == "read": + docs._resolve_doc(request["path"]) + key = _digest({"request": request, "active_tokens": state["active_tokens"]}) + if key in state["requests"]: + stored = state["requests"][key] + result = stored.get("result", stored) + expected = _digest({"session_id": session_id, "request": request, + "active_tokens": state["active_tokens"], "result": result}) + content = result.get("model_content") + if not isinstance(content, str) or hashlib.sha256(content.encode()).hexdigest() != result.get("model_content_sha256") \ + or stored.get("receipt_sha256") != expected \ + or self._count(content) != result.get("delivery_tokens") \ + or not self._count(content) <= result.get("consumed_tokens", -1) <= state["consumed_tokens"]: + raise ContextError("cached context receipt/content integrity check failed") + for reference in result["documents"]: + self.view.path(reference["path"]) + return result + units, extra = build(docs) + rendered, references, truncated = [], [], False + for unit in units: + budget_partial = False + content = unit.pop("content") + unit_hash = _digest({"repository": binding["repo_id"], "path": unit["path"], "content": content}) + if unit_hash in state["units"]: + previous = state["units"][unit_hash] + if previous["complete"]: + continue + content = content[previous["delivered_chars"]:] + unit["continuation"] = True + else: + previous = {"delivered_chars": 0} + pins = unit.get("source_pins") or [binding["source_pin"]] + def render(value): + injected = {**unit, "repo_id": binding["repo_id"], "source_pins": pins, "content": value} + serialized = json.dumps(injected, ensure_ascii=False).replace("<", "\\u003c") + return "\n\n" + serialized + "\n\n" + # Headers and rendered delimiters consume the same budget. + remaining = state["active_tokens"] - state["consumed_tokens"] + full = render(content) + if self._count(full) > remaining: + budget_partial = True + unit["partial"] = True + # A sliced facet is not an intact injected facet. + unit["included_facets"] = [] + for metadata in ("included_facet_acceptance_modes", "included_validation_kinds", "included_facet_basis"): + if metadata in unit: + unit[metadata] = {} + if "available_facets" in unit: + unit["not_injected_facets"] = list(unit["available_facets"]) + low, high = 0, len(content) + while low < high: + mid = (low + high + 1) // 2 + if self._count(render(content[:mid])) <= remaining: + low = mid + else: + high = mid - 1 + if not low: + truncated = True + break + content, full, truncated = content[:low], render(content[:low]), True + state["consumed_tokens"] += self._count(full) + state["units"][unit_hash] = {"delivered_chars": previous["delivered_chars"] + len(content), + "complete": not budget_partial} + references.append({**unit, "repo_id": binding["repo_id"], "content_sha256": hashlib.sha256(content.encode()).hexdigest()}) + rendered.append(full) + if state["consumed_tokens"] >= state["active_tokens"]: + break + result = {"status": "ready" if rendered else "budget_exhausted" if truncated or state["consumed_tokens"] >= state["active_tokens"] else "no_new_context", + "session_id": session_id, "documents": references, "model_content": "".join(rendered), + "content_chars": sum(len(s) for s in rendered), "truncated": truncated, + "delivery_tokens": self._count("".join(rendered)), + "consumed_tokens": state["consumed_tokens"], + "remaining_tokens": state["active_tokens"] - state["consumed_tokens"], + "token_accounting": self.tokenizer_id, "actual_provider_usage": "unknown", **extra} + result["model_content_sha256"] = hashlib.sha256(result["model_content"].encode()).hexdigest() + if request["op"] == "read" and references and references[-1].get("partial"): + result["next_offset"] = request["offset"] + state["units"][unit_hash]["delivered_chars"] + state["requests"][key] = {"request": request, "result": result, + "receipt_sha256": _digest({"session_id": session_id, "request": request, + "active_tokens": state["active_tokens"], "result": result})} + self._save(db, session_id, state) + return result + + def related(self, session_id: str, changed_files: list[str], *, query: str = "", + repository: str | None = None) -> dict: + def build(docs): + result = docs.related(changed_files, query=query, max_documents=100, + max_page_chars=65536, max_content_chars=262144, max_changed_files=None) + return result["documents"], {"retrieval_status": result["status"], "guidance": result["guidance"]} + return self._deliver(session_id, {"op": "related", "changed_files": changed_files, + "query": query, "repository": repository}, build) + + def search(self, session_id: str, query: str, *, repository: str | None = None, limit: int = 40) -> dict: + def build(docs): + matches = docs.search(query, limit=limit) + return [{"path": h["path"], "line": h["line"], "content": h["text"]} for h in matches], {} + return self._deliver(session_id, {"op": "search", "query": query, + "repository": repository, "limit": limit}, build) + + def read(self, session_id: str, path: str, *, offset: int = 0, + limit: int = 65536, repository: str | None = None) -> dict: + def build(docs): + page = docs.read(path, offset=offset, limit=limit) + return [{"path": page["path"], "offset": offset, "content": page["content"]}], {"next_offset": page["next_offset"]} + return self._deliver(session_id, {"op": "read", "path": path, "offset": offset, + "limit": limit, "repository": repository}, build) + + def inject(self, session_id: str, documents: list[dict], *, purpose: str) -> dict: + """Budget provider-selected procedures/rules through the same ledger. + + The host passes only document IDs, never arbitrary unverified prose. + """ + paths = [row["path"] for row in documents] + def build(docs): + units = [] + for row in documents: + path = row["path"] + page = docs.read(path, limit=65536) + content = page["content"] + if row.get("kind") == "direct_map": + from .direct_routing import _direct_quick_map_text + content, status = _direct_quick_map_text(content) + if status == "unavailable": + continue + units.append({"path": path, "content": content, "purpose": purpose}) + return units, {} + return self._deliver(session_id, {"op": "inject", "documents": documents, "purpose": purpose}, build) diff --git a/src/infermatrix_copilot/knowledge_docs.py b/src/infermatrix_copilot/knowledge_docs.py index dd3ed5b76..d60f98118 100644 --- a/src/infermatrix_copilot/knowledge_docs.py +++ b/src/infermatrix_copilot/knowledge_docs.py @@ -5,6 +5,7 @@ import fnmatch import re from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from typing import Callable @@ -17,6 +18,17 @@ class KnowledgeDocsError(ValueError): """A refused or invalid knowledge-base operation.""" +@lru_cache(maxsize=256) +def _parsed_explanation(raw: str): + """Content-keyed parsing only; every caller still validates snapshot bytes. + + A changed development file has a different key. This deliberately does not + cache path reads or bypass KnowledgeView.path's integrity checks. + """ + page = Page.parse(raw) + return page, page.frontmatter_data(), depth_sections(raw), visible_text(raw[len(page.frontmatter):]).strip() + + @dataclass(frozen=True) class KnowledgeHit: path: str @@ -144,7 +156,9 @@ def search(self, query: str, *, limit: int = 40) -> list[dict]: hits.sort(key=lambda h: (-h.score, h.path, h.line)) return [hit.as_dict() for hit in hits[:limit]] - def related(self, changed_files: list[str], *, query: str = "") -> dict: + def related(self, changed_files: list[str], *, query: str = "", + max_documents: int = 2, max_content_chars: int = 6000, + max_page_chars: int = 3000, max_changed_files: int | None = 100) -> dict: """Select at most two explanatory documents with 6k characters of prose. Source/entry-point matches outrank declared globs and description hints. @@ -152,10 +166,18 @@ def related(self, changed_files: list[str], *, query: str = "") -> dict: """ if not isinstance(changed_files, list) or any(not isinstance(p, str) for p in changed_files): raise KnowledgeDocsError("changed files must be a list of repository-relative paths") + if any(type(value) is not int or value < 1 for value in + (max_documents, max_content_chars, max_page_chars)): + raise KnowledgeDocsError("related limits must be positive integers") + if max_documents > 100 or max_content_chars > 262144 or max_page_chars > 65536: + raise KnowledgeDocsError("related limits exceed the supported window") changed = list(dict.fromkeys(p.replace("\\", "/") for p in changed_files)) if any(not _source_path(p) for p in changed): raise KnowledgeDocsError("changed files must be repository-relative paths") - changed = changed[:100] + if max_changed_files is not None: + if type(max_changed_files) is not int or max_changed_files < 1: + raise KnowledgeDocsError("max_changed_files must be a positive integer or None") + changed = changed[:max_changed_files] query = str(query)[:12000].casefold() terms = set(re.findall(r"[\w.-]{3,}", query)) candidates, warnings = [], [] @@ -172,14 +194,12 @@ def related(self, changed_files: list[str], *, query: str = "") -> dict: if "\n(.*?)\n", re.S) diff --git a/src/infermatrix_copilot/knowledge_service/pinned_claims.py b/src/infermatrix_copilot/knowledge_service/pinned_claims.py index 4842bc2f5..3ecc53c27 100644 --- a/src/infermatrix_copilot/knowledge_service/pinned_claims.py +++ b/src/infermatrix_copilot/knowledge_service/pinned_claims.py @@ -35,7 +35,7 @@ from .facts import FactsError, claims_in, describe, holds, merge_claims, observe -_SHA = re.compile(r"[0-9a-f]{40}") +_SHA = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") def _gh_pull(repository: str, number: int) -> dict: diff --git a/src/infermatrix_copilot/sdk/v1/__init__.py b/src/infermatrix_copilot/sdk/v1/__init__.py index d6846e685..7de1f23b8 100644 --- a/src/infermatrix_copilot/sdk/v1/__init__.py +++ b/src/infermatrix_copilot/sdk/v1/__init__.py @@ -1,6 +1,7 @@ """InferMatrixCopilot embedded SDK v1 public surface.""" from .direct import DirectClient, get_capabilities +from ...knowledge_context import ContextBudget from .knowledge import KnowledgeCurator, KnowledgeValidatorError from .models import ( DIRECT_API_VERSION, @@ -56,6 +57,7 @@ "STRICT_API_VERSION", "TRACE_SCHEMA", "Capabilities", + "ContextBudget", "ChangedPath", "CarriedFinding", "FindingRecheck", diff --git a/src/infermatrix_copilot/sdk/v1/direct.py b/src/infermatrix_copilot/sdk/v1/direct.py index 086300bd4..a902ec943 100644 --- a/src/infermatrix_copilot/sdk/v1/direct.py +++ b/src/infermatrix_copilot/sdk/v1/direct.py @@ -90,13 +90,11 @@ def get_capabilities( from ...knowledge_view import KnowledgeView - knowledge = KnowledgeView.current().root + view = KnowledgeView.current() + knowledge = view.root adapters = adapters_root() - repositories = tuple(sorted( - path.name - for path in (knowledge / "repos").iterdir() - if path.is_dir() and (path / "_index.md").is_file() - )) + from ...kb_service.repo_spec import list_snapshot_repos + repositories = tuple(binding.repo_id for binding in list_snapshot_repos(view)) locking = bool(supports_file_locking) return Capabilities( distribution_version=_distribution_version(), @@ -134,13 +132,19 @@ class DirectClient: cannot grow memory without limit. """ - def __init__(self, *, max_issued_contexts: int = _DEFAULT_CONTEXT_LIMIT) -> None: + def __init__(self, *, max_issued_contexts: int = _DEFAULT_CONTEXT_LIMIT, + knowledge_context_path: str | Path | None = None, + allowed_knowledge_repositories: tuple[str, ...] = ()) -> None: if max_issued_contexts < 1: raise InvalidRequestError("max_issued_contexts must be >= 1") self._adapters = adapters_root() self._max_issued_contexts = int(max_issued_contexts) self._issued_contexts: OrderedDict[str, _IssuedContext] = OrderedDict() self._context_lock = threading.Lock() + self._knowledge_context_path = Path(knowledge_context_path) if knowledge_context_path else \ + Path.home() / ".infermatrix-copilot" / "knowledge-context.sqlite" + self._knowledge_context_services: dict[str, Any] = {} + self._allowed_knowledge_repositories = tuple(allowed_knowledge_repositories) @staticmethod def _view(): @@ -250,7 +254,7 @@ def read_document( next_offset=next_offset, ) - def _request_values(self, request: DirectReviewRequest) -> tuple[str, list[str]]: + def _request_values(self, request: DirectReviewRequest, *, allow_sha256: bool = False) -> tuple[str, list[str]]: alias = request.repository.alias.strip() if not request.review_id.strip(): raise InvalidRequestError("review_id must not be empty") @@ -258,11 +262,14 @@ def _request_values(self, request: DirectReviewRequest) -> tuple[str, list[str]] raise InvalidRequestError("pr_number must be >= 0") if not alias: raise InvalidRequestError("repository alias must not be empty") - if alias.replace("_", "-").casefold() not in { + from ...kb_service.repo_spec import resolve_snapshot_repo + binding = resolve_snapshot_repo(self._view(), alias) + if binding is None and alias.replace("_", "-").casefold() not in { item.casefold() for item in self.capabilities().supported_repositories }: raise UnsupportedRepositoryError(f"unsupported repository: {alias!r}") - if not _FULL_SHA.fullmatch(request.expected_head_sha.strip()): + head = request.expected_head_sha.strip() + if not _FULL_SHA.fullmatch(head) and not (allow_sha256 and re.fullmatch(r"[0-9a-fA-F]{64}", head)): raise InvalidRequestError( "expected_head_sha must be exactly 40 hexadecimal characters" ) @@ -278,6 +285,87 @@ def _request_values(self, request: DirectReviewRequest) -> tuple[str, list[str]] changed_files.append(path.as_posix()) return alias, changed_files + def open_knowledge_context(self, request: DirectReviewRequest, *, budget=None) -> dict: + """Opt into the cumulative adaptive protocol; the v1 plan stays unchanged.""" + from ...knowledge_context import KnowledgeContextService + alias, _ = self._request_values(request, allow_sha256=True) + view = self._view() + service = KnowledgeContextService(view, self._knowledge_context_path, + allowed_repositories=self._allowed_knowledge_repositories) + status = service.open_session(alias, source_pin=request.expected_head_sha.casefold(), + review_id=request.review_id, budget=budget) + self._knowledge_context_services[status["session_id"]] = service + return status + + def _knowledge_context(self, session_id: str): + service = self._knowledge_context_services.get(session_id) + if service is None: + import sqlite3 + from ...knowledge_context import KnowledgeContextService + from ...knowledge_view import KnowledgeView, _load_view + if not re.fullmatch(r"[0-9a-f]{64}", session_id) or not self._knowledge_context_path.is_file(): + raise InvalidRequestError("unknown knowledge session") + with sqlite3.connect(self._knowledge_context_path) as db: + row = db.execute("SELECT state FROM sessions WHERE id=?", (session_id,)).fetchone() + if row is None: + raise InvalidRequestError("unknown knowledge session") + identity = json.loads(row[0])["identity"]["view"] + root = Path(identity["root"]) + if (root.parent / "MANIFEST.json").is_file(): + view = _load_view(str(root.parent)) + else: + view = KnowledgeView(root, identity["snapshot"]) + service = KnowledgeContextService(view, self._knowledge_context_path, + allowed_repositories=self._allowed_knowledge_repositories) + service.status(session_id) # validate pinned bytes before resuming + self._knowledge_context_services[session_id] = service + return service + + def plan_adaptive(self, request: DirectReviewRequest, *, budget=None) -> dict: + """Return one budgeted model packet, with references instead of SDK excerpts.""" + from ...direct_routing import direct_review_plan + alias, changed = self._request_values(request, allow_sha256=True) + status = self.open_knowledge_context(request, budget=budget) + service, session_id = self._knowledge_context(status["session_id"]), status["session_id"] + raw = direct_review_plan(alias, title=request.title, body=request.body, + changed_files=changed, diff=request.diff, view=service.view) + routes = [{"path": service.view.relative(row["path"]), "kind": "direct_map"} + for row in raw["knowledge_routes"]] + procedures = [{"path": service.view.relative(path)} for path in raw["mandatory_review_guides"]] + guide_packet = service.inject(session_id, procedures + routes, purpose="review-procedures-and-owner-routes") + related = service.related(session_id, changed, query="\n".join((request.title, request.body, request.diff))) + # There is exactly one injectable copy of every returned content fragment. + content = guide_packet["model_content"] + related["model_content"] + context_id = _sha256(json.dumps({"session_id": session_id, "request": request.to_dict()}, + sort_keys=True).encode()) + self._remember_context(context_id, request.expected_head_sha.casefold(), + tuple(row.to_dict() for row in request.carried_findings), service.view) + return {"protocol_version": "knowledge-context-1", "session_id": session_id, + "review_context_id": context_id, + "expected_head_sha": request.expected_head_sha.casefold(), + "model_content": content, "model_content_sha256": hashlib.sha256(content.encode()).hexdigest(), + "documents": guide_packet["documents"] + related["documents"], + "budget": service.status(session_id), "routing": raw["routing"], + "execution_budget": raw["execution_budget"], + "first_review_checklist": raw["first_review_checklist"], + "progress_update": raw["progress_update"], "completion_gate": raw["completion_gate"], + "guidance": related.get("guidance", ""), + "truncated": guide_packet["truncated"] or related["truncated"]} + + def read_knowledge_context(self, session_id: str, document_id: str, *, offset: int = 0, + repository: str | None = None) -> dict: + return self._knowledge_context(session_id).read(session_id, document_id, offset=offset, repository=repository) + + def search_knowledge_context(self, session_id: str, query: str, *, repository: str | None = None) -> dict: + return self._knowledge_context(session_id).search(session_id, query, repository=repository) + + def related_knowledge_context(self, session_id: str, changed_files: list[str], *, query: str = "", + repository: str | None = None) -> dict: + return self._knowledge_context(session_id).related(session_id, changed_files, query=query, repository=repository) + + def expand_knowledge_context(self, session_id: str, *, target_tokens: int, reason: str) -> dict: + return self._knowledge_context(session_id).expand(session_id, target_tokens=target_tokens, reason=reason) + def _remember_context( self, context_id: str, expected_head_sha: str, carried=(), view=None, ) -> None: @@ -325,7 +413,9 @@ def plan(self, request: DirectReviewRequest) -> DirectReviewPlan: ) for item in raw.get("knowledge_routes") or [] ) - repo_name = alias.replace("_", "-") + from ...kb_service.repo_spec import resolve_snapshot_repo + binding = resolve_snapshot_repo(view, alias) + repo_name = binding.knowledge_slice.removeprefix("repos/") if binding else alias.replace("_", "-") map_candidates = ( view.root / "README.md", view.root / "general" / "_index.md", diff --git a/src/infermatrix_copilot/sdk/v1/models.py b/src/infermatrix_copilot/sdk/v1/models.py index 731ac7dae..32ce899a5 100644 --- a/src/infermatrix_copilot/sdk/v1/models.py +++ b/src/infermatrix_copilot/sdk/v1/models.py @@ -320,8 +320,16 @@ class StrictRuntimeConfig(_Serializable): max_workers: int = 1 def __post_init__(self) -> None: - if not self.repository.alias or self.repository.full_name.count("/") != 1: - raise InvalidRequestError("Strict runtime needs a repository alias and full name") + if not self.repository.alias: + raise InvalidRequestError("Strict runtime needs a repository alias") + if self.repository.full_name: + if "/" not in self.repository.full_name or any(not part for part in self.repository.full_name.split("/")): + raise InvalidRequestError("Strict repository namespace must have non-empty path segments") + else: + from ...kb_service.repo_spec import resolve_snapshot_repo + from ...knowledge_view import KnowledgeView + if resolve_snapshot_repo(KnowledgeView.current(), self.repository.alias) is None: + raise InvalidRequestError("Strict runtime needs a registered repository or forge full name") if not self.checkout_path or not self.allowed_root: raise InvalidRequestError("Strict runtime needs a checkout and allowed root") if (isinstance(self.max_workers, bool) diff --git a/src/infermatrix_copilot/thin_mcp_server.py b/src/infermatrix_copilot/thin_mcp_server.py index 88b40b565..12ff4ee16 100644 --- a/src/infermatrix_copilot/thin_mcp_server.py +++ b/src/infermatrix_copilot/thin_mcp_server.py @@ -210,6 +210,9 @@ def build_mcp( from mcp.types import ToolAnnotations core = core or CopilotMCP(settings) + from .sdk.v1.direct import DirectClient + context_client = DirectClient(knowledge_context_path=core.settings.run_root / "knowledge-context.sqlite", + allowed_knowledge_repositories=core.settings.allowed_knowledge_repositories) # Approval-gating hosts (codex asks per-call approval for tools without # hints, and cancels them outright in headless runs) read these @@ -274,6 +277,7 @@ def review( expected_head_sha: str = "", idempotency_key: str = "", diff: str = "", + knowledge_profile: str = "legacy", ) -> dict: """Begin a Direct or Strict review. @@ -310,6 +314,14 @@ def run() -> dict: if selected_mode not in {"direct", "strict"}: raise ValueError("mode must be 'direct' or 'strict'") if selected_mode == "direct": + if knowledge_profile == "adaptive": + from .sdk.v1.models import ChangedPath, DirectReviewRequest, RepositoryRef + request = DirectReviewRequest(idempotency_key or f"direct:{target}", RepositoryRef(repo), + 0, expected_head_sha, title, body, + tuple(ChangedPath(path) for path in changed_files or ()), diff=diff) + return context_client.plan_adaptive(request) + if knowledge_profile != "legacy": + raise ValueError("knowledge_profile must be legacy or adaptive") return direct_review_plan( repo, title=title, @@ -480,6 +492,41 @@ def run() -> dict: return _guard(run) + @mcp.tool(annotations=read_only) + def open_knowledge_context(repo: str, source_pin: str, review_id: str, + model_context_tokens: int = 131072, + source_reserve_tokens: int = 32000, + output_reserve_tokens: int = 8192) -> dict: + """Open an explicit adaptive 24k/64k session; supplied model capacity is configuration, not detected usage.""" + from .sdk.v1.models import DirectReviewRequest, RepositoryRef + from .knowledge_context import ContextBudget + def run(): + request = DirectReviewRequest(review_id, RepositoryRef(repo), 0, source_pin, "", "", ()) + return context_client.open_knowledge_context(request, budget=ContextBudget( + model_context_tokens=model_context_tokens, source_reserve_tokens=source_reserve_tokens, + output_reserve_tokens=output_reserve_tokens)) + return _guard(run) + + @mcp.tool(annotations=read_only) + def read_knowledge_context(session_id: str, path: str, offset: int = 0, repository: str | None = None) -> dict: + """Read source-pinned knowledge using the session's remaining cumulative budget.""" + return _guard(lambda: context_client.read_knowledge_context(session_id, path, offset=offset, repository=repository)) + + @mcp.tool(annotations=read_only) + def search_knowledge_context(session_id: str, query: str, repository: str | None = None) -> dict: + """Search snippets consume the same budget as full knowledge reads.""" + return _guard(lambda: context_client.search_knowledge_context(session_id, query, repository=repository)) + + @mcp.tool(annotations=read_only) + def related_knowledge_context(session_id: str, changed_files: list[str], query: str = "", repository: str | None = None) -> dict: + """Select additional intact feature contexts within the shared remaining budget.""" + return _guard(lambda: context_client.related_knowledge_context(session_id, changed_files, query=query, repository=repository)) + + @mcp.tool(annotations=read_only) + def expand_knowledge_context(session_id: str, target_tokens: int, reason: str) -> dict: + """Request more context, with a concrete unresolved contract; never consumes reserved source/output capacity.""" + return _guard(lambda: context_client.expand_knowledge_context(session_id, target_tokens=target_tokens, reason=reason)) + return mcp diff --git a/test/test_evidence_bundle.py b/test/test_evidence_bundle.py new file mode 100644 index 000000000..a8c3b2cff --- /dev/null +++ b/test/test_evidence_bundle.py @@ -0,0 +1,39 @@ +import hashlib +from types import SimpleNamespace + +import pytest + +from infermatrix_copilot.kb_service.evidence_bundle import ( + affected_features, build_evidence_bundle, materialize_bundle, +) + + +def bundle(tmp_path): + path = tmp_path / 'library.py' + path.write_text('def exported():\n return 1\n') + idx = SimpleNamespace(identity={'pin': 'a' * 40, 'scope': {'roots': ['.']}}, + sha256='b' * 64, entries={'library.py': {'status': 'ready', 'kind': 'source', + 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'lines': path.read_text().splitlines()}}) + ref = {'path': 'library.py', 'start': 1, 'end': 2} + return build_evidence_bundle(idx, 'exported', catalog_hash='c' * 64, + refs=[ref, ref], unit_ids=['one', 'one']) + + +def test_discovery_handoff_contains_exact_source_span(tmp_path): + value = bundle(tmp_path) + result = materialize_bundle(value, tmp_path, pin='a' * 40, catalog_hash='c' * 64) + assert len(value['refs']) == 1 and value['unit_ids'] == ['one'] + assert result[0]['text'] == '1: def exported():\n2: return 1' + assert affected_features({'exported': value}, ['library.py']) == ['exported'] + + +@pytest.mark.parametrize('change', ['body', 'source', 'pin', 'catalog']) +def test_changed_discovery_handoff_fails_closed(tmp_path, change): + value = bundle(tmp_path) + pin, catalog = 'a' * 40, 'c' * 64 + if change == 'body': value['feature_id'] = 'other' + if change == 'source': (tmp_path / 'library.py').write_text('return 2\n') + if change == 'pin': pin = 'd' * 40 + if change == 'catalog': catalog = 'd' * 64 + with pytest.raises(ValueError): + materialize_bundle(value, tmp_path, pin=pin, catalog_hash=catalog) diff --git a/test/test_kb_depth_retrieval_audit.py b/test/test_kb_depth_retrieval_audit.py index 0211fa620..b422aec42 100644 --- a/test/test_kb_depth_retrieval_audit.py +++ b/test/test_kb_depth_retrieval_audit.py @@ -105,10 +105,33 @@ def test_long_facet_is_partial_injection_and_remains_a_budgeted_followup(auditor assert case["facet_injection"]["api"] == "partial" document = case["documents"][0] assert document["content_chars"] <= 3000 + assert document["partial"] is True + assert document["included_facets"] == [] + assert document["included_facet_basis"] == {} + assert document["available_facets"] == document["not_injected_facets"] == ["api"] assert case["followup_read_paths"] == [document["path"]] assert report["summary"]["probes"]["query_only"]["target_feature_injection"]["full"] == 0 +@pytest.mark.parametrize("content", ["", "Unrelated prose."]) +def test_partial_metadata_without_real_section_prose_is_not_depth(auditor, world, monkeypatch, content): + root, _ = world(1, long=True) + direct = auditor.direct_review_plan + + def missing_partial_content(*args, **kwargs): + plan = direct(*args, **kwargs) + selected = plan["related_knowledge"]["documents"][0] + assert selected["partial"] and selected["included_facets"] == [] + selected["content"] = content + plan["related_knowledge"]["content_chars"] = len(content) + return plan + + monkeypatch.setattr(auditor, "direct_review_plan", missing_partial_content) + report = auditor.audit_retrieval(root, "retrievaldemo", PIN, require_depth=True) + assert report["summary"]["probes"]["query_only"]["depth_hits"] == 0 + assert any("feature depth absent from actual Direct context" in p for p in report["problems"]) + + def test_verified_test_gap_is_visible_without_claiming_tests_passed(auditor, world): root, _ = world(1, absent=True) report = auditor.audit_retrieval(root, "retrievaldemo", PIN, require_depth=True) diff --git a/test/test_kb_discovery_contract_units.py b/test/test_kb_discovery_contract_units.py new file mode 100644 index 000000000..2a4723cab --- /dev/null +++ b/test/test_kb_discovery_contract_units.py @@ -0,0 +1,234 @@ +"""Autonomous entry leads, saturated inputs and truthful residual checkpoints.""" +import hashlib +import json +from types import SimpleNamespace + +import pytest + +from infermatrix_copilot.kb_service.feature_discovery_index import ( + build_discovery_index, contract_units, unit_evidence, + build_for_stage, +) +from infermatrix_copilot.kb_service.init_feature_discovery import DiscoveryEngine, _hash +from infermatrix_copilot.kb_service.init_budget import BudgetExhausted +from infermatrix_copilot.kb_service.models import ModelReply, ModelRole + +PIN = "a" * 40 + + +def make_index(root, files): + for name, text in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text if isinstance(text, bytes) else text.encode()) + return build_discovery_index(root, pin=PIN, scope={"roots": ["src/"], "exclude": []}, doc_globs=["docs/*.md"]) + + +def payload(prompt): + return json.loads(prompt.split("\n", 1)[1].rsplit("\n", 1)[0]) + + +def row(key, path="src/library.py", start=1, end=2): + return {"id": key, "title": key, "description": "Return an explicit caller value.", "owner": "core", + "relation": "new", "related_id": "", "aliases": [], + "evidence": [{"path": path, "start": start, "end": end}]} + + +def reply(role, data): + text = json.dumps(data) + model = ModelRole(role, "zcode" if role == "generator" else "codex", "GLM-5.3" if role == "generator" else "gpt-6-sol") + return ModelReply(model, data, text, model.model, {}, .1, None, "native-test", + hashlib.sha256(text.encode()).hexdigest()) + + +def judge(data, supported="yes"): + return {"decisions": {r["id"]: {"supported": supported, "relation": "new", "related_id": "", + "reason": "Supplied implementation establishes the explicit value."} + for r in data["candidates"]}} + + +def test_multilanguage_entries_do_not_require_document_keywords(tmp_path): + index = make_index(tmp_path, { + "src/library.py": "def public(value):\n return _helper(value)\n\ndef _helper(value):\n return value\n", + "src/plugin.ts": "// implementation\nexport const run = (value: string) => value;\nrouter.post('/start', run);\n", + "src/library.unknown": "publish values to caller\n", + "src/broken.py": b"\xff", + "sdks/nested/tests/test_public.py": "def test_public():\n assert public(1) == 1\n", + }) + units = contract_units(index) + assert {u["path"] for u in units} == set(index.production) + assert any(u["name"] == "run" for u in units) + assert any(u["kind"] == "registration" for u in units) + assert any(u["language"] == "unknown" for u in units) + unreadable = next(u for u in units if u["path"] == "src/broken.py") + assert unreadable["status"] == "error" and unit_evidence(index, unreadable) == [] + public = next(u for u in units if u["name"] == "public") + excerpts = unit_evidence(index, public) + assert any("def _helper" in e["text"] for e in excerpts) + assert any(e["path"] == "sdks/nested/tests/test_public.py" and "assert" in e["text"] for e in excerpts) + assert sum(len(e["text"]) + 150 for e in excerpts) <= 12000 + + +def test_unknown_candidate_cannot_hide_another_entry_in_the_same_file(tmp_path): + index = make_index(tmp_path, {"src/library.py": "def first():\n return 1\n\ndef second():\n return 2\n"}) + first, second = row("first"), row("second", start=4, end=5) + for r in (first, second): + r.update(generator_receipts=[{"trace_id": "g"}], origin_rounds=["source"]) + reviews = {"first": {**judge({"candidates": [first]})["decisions"]["first"], "candidate_sha256": _hash(first), "judge_receipt": {"trace_id": "j"}}, + "second": {"supported": "unsure", "relation": "unknown", "reason": "Unknown boundary", "attempts": 4}} + engine = DiscoveryEngine(index, seeds=[], owners=[], state={"candidates": {r["id"]: r for r in (first, second)}, "reviews": reviews}, call=None, save=lambda: None) + resolution = engine.unit_resolution() + assert resolution["associated"] == 1 + assert [u["name"] for u in resolution["unassociated_units"]] == ["second"] + + +def test_broad_accepted_reference_cannot_hide_unexplained_entries(tmp_path): + index = make_index(tmp_path, {"src/library.py": "def first():\n return 1\n\ndef second():\n return 2\n"}) + candidate = row("first", end=5) + candidate.update(description="The first public contract returns one.", + generator_receipts=[{"trace_id": "g"}], origin_rounds=["source"]) + review = {**judge({"candidates": [candidate]})["decisions"]["first"], + "candidate_sha256": _hash(candidate), "judge_receipt": {"trace_id": "j"}} + engine = DiscoveryEngine(index, seeds=[], owners=[], + state={"candidates": {"first": candidate}, "reviews": {"first": review}}, + call=None, save=lambda: None) + resolution = engine.unit_resolution() + assert resolution["associated"] == 1 + assert [u["name"] for u in resolution["unassociated_units"]] == ["second"] + + +def test_saturated_packet_is_split_and_resume_does_not_resample(tmp_path): + index = make_index(tmp_path, {"src/library.py": "\n".join(f"value_{n} = {n}" for n in range(64))}) + sizes = [] + def call(role, system, prompt, validate): + data = payload(prompt) + lines = sum(e["end"] - e["start"] + 1 for e in data["files"]) + sizes.append(lines) + anchor = data["files"][0] + result = {"candidates": [row(f"entry-{n}", start=anchor["start"], end=anchor["start"]) + for n in range(24)]} + validate(result) + return reply(role, result) + state = {} + engine = DiscoveryEngine(index, seeds=[], owners=[], state=state, call=call, save=lambda: None, concurrency=1) + assert engine.scan() + assert sorted(sizes) == [16, 16, 16, 16, 32, 32, 64] + assert sum(bool(t.get("overflow_unknown")) for t in state["tasks"].values()) == 4 + assert max(t["split_depth"] for t in state["tasks"].values()) == 2 + resumed = DiscoveryEngine(index, seeds=[], owners=[], state=state, + call=lambda *a, **k: pytest.fail("successful task was sampled again"), save=lambda: None) + assert resumed.scan() + + +def test_residual_pass_finds_unexplained_entry_and_preserves_four_round_cap(tmp_path): + index = make_index(tmp_path, {"src/library.py": "def first():\n return 1\n\ndef second():\n return 2\n"}) + calls = [] + def call(role, system, prompt, validate): + data = payload(prompt) + calls.append((role, data.get("round"))) + if role == "generator": + result = {"candidates": [row("second", start=4, end=5) if data["round"].startswith("residual") else row("first")]} + else: + result = judge(data) + validate(result) + return reply(role, result) + engine = DiscoveryEngine(index, seeds=[], owners=[], state={}, call=call, save=lambda: None) + assert engine.scan() and engine.review() and engine.supplemental_scan() + assert engine.unit_resolution()["associated"] == 2 + assert set(engine.state["residual_rounds"]) == {"1", "2"} + assert sum(role == "generator" and (kind or "").startswith("residual") for role, kind in calls) == 1 + assert engine.state["residual_complete"] + frozen = json.loads(json.dumps(engine.state)) + resumed = DiscoveryEngine(index, seeds=[], owners=[], state=frozen, + call=lambda *a, **k: pytest.fail("completed residual task was sampled again"), save=lambda: None) + assert resumed.scan() and resumed.review() and resumed.supplemental_scan() + + +def test_budget_stop_preserves_residual_plan_and_completed_source(tmp_path): + index = make_index(tmp_path, {"src/library.py": "def first():\n return 1\n\ndef second():\n return 2\n"}) + blocked = True + calls = [] + def call(role, system, prompt, validate): + data = payload(prompt) + calls.append(data.get("round", "judge")) + if role == "generator" and data["round"].startswith("residual") and blocked: + raise BudgetExhausted("save checkpoint") + result = {"candidates": [row("first") if data.get("round") == "source" else row("second", start=4, end=5)]} if role == "generator" else judge(data) + return reply(role, result) + state = {} + first = DiscoveryEngine(index, seeds=[], owners=[], state=state, call=call, save=lambda: None, concurrency=1) + assert first.scan() and first.review() and not first.supplemental_scan() + assert not state["residual_rounds"]["1"]["complete"] + assert any(t["status"] == "pending" for t in state["tasks"].values()) + blocked = False + resumed = DiscoveryEngine(index, seeds=[], owners=[], state=state, call=call, save=lambda: None) + assert resumed.scan() and resumed.review() and resumed.supplemental_scan() + assert calls.count("source") == 1 and resumed.unit_resolution()["associated"] == 2 + + +def test_exhausted_candidate_with_new_evidence_does_not_reset_review_counter(tmp_path): + index = make_index(tmp_path, {"src/library.py": "def first():\n return 1\n\ndef second():\n return 2\n"}) + calls = [] + def call(role, system, prompt, validate): + data = payload(prompt) + calls.append(role) + result = {"candidates": [row("first", end=5 if data["round"].startswith("residual") else 2)]} if role == "generator" else judge(data, "unsure") + return reply(role, result) + state = {} + engine = DiscoveryEngine(index, seeds=[], owners=[], state=state, call=call, save=lambda: None) + assert engine.scan() and engine.review() + assert calls.count("judge") == 4 + assert engine.supplemental_scan() + assert calls.count("judge") == 4 and state["review_attempts"]["first"] == 4 + assert engine.unit_resolution()["associated"] == 0 + + +def test_discovery_index_accepts_sha256_commit_identity(tmp_path): + path = tmp_path / "src/library.py" + path.parent.mkdir() + path.write_text("def public(): return 1\n") + index = build_discovery_index(tmp_path, pin="a" * 64, scope={"roots": ["src/"], "exclude": []}) + assert index.identity["pin"] == "a" * 64 + + +def test_portable_scope_uses_approved_suffixes_and_custom_tests(tmp_path): + from infermatrix_copilot.kb_service.config import InitConfig + tree, state = tmp_path / "repo", tmp_path / "state" + make_index(tree, {"src/library.fiction": "publish caller value\n", + "src/unused.py": "def outside_suffix(): pass\n", + "extra/unapproved.fiction": "unapproved first party contract\n", + "qa/check_contract.fiction": "check caller value\n", + "src/vendor/tests/imported.py": "assert copied_behavior()\n"}) + stage = SimpleNamespace( + lifecycle=SimpleNamespace(init=InitConfig(source_roots=("src/",), exclude=("qa/**", "src/vendor/**")), repo="demo"), + record=SimpleNamespace(pin=PIN), manifest={"portable_scope": {"suffixes": [".fiction"], "filenames": [], "test_globs": ["qa/**"]}}, + _coverage_policy_path=lambda: "coverage.yaml", _base_sha="b" * 40, + rt=SimpleNamespace(state_dir=state, knowledge=SimpleNamespace(show=lambda *a: None))) + index = build_for_stage(tree, stage) + assert index.production == ["src/library.fiction"] + assert index.tests == ["qa/check_contract.fiction"] + assert index.identity["scope"]["suffixes"] == [".fiction"] + assert index.identity["scope"]["test_globs"] == ["qa/**"] + assert index.scope_suggestions == [ + {"path": "extra/unapproved.fiction", "reason": "code outside declared production roots"}, + {"path": "src/unused.py", "reason": "unsupported suffix outside declared suffix scope"}] + + +def test_incremental_paths_limit_work_but_do_not_shrink_inventory(tmp_path): + index = make_index(tmp_path, {"src/changed.py": "def changed():\n return 1\n", + "src/untouched.py": "def untouched():\n return 2\n", + "docs/guide.md": "# Untouched capability\n"}) + offered = [] + def call(role, system, prompt, validate): + data = payload(prompt) + offered.extend(e["path"] for e in data["files"]) + return reply(role, {"candidates": []}) + engine = DiscoveryEngine(index, seeds=[], owners=[], state={}, call=call, save=lambda: None, + scan_paths=["src/changed.py"]) + assert engine.scan() and engine.review() and engine.supplemental_scan() + assert set(offered) == {"src/changed.py"} + assert index.production == ["src/changed.py", "src/untouched.py"] + assert {u["path"] for u in engine.units} == {"src/changed.py"} + with pytest.raises(ValueError, match="outside the frozen index"): + DiscoveryEngine(index, seeds=[], owners=[], state={}, call=None, save=lambda: None, + scan_paths=["src/deleted.py"]) diff --git a/test/test_kb_feature_discovery.py b/test/test_kb_feature_discovery.py index 4c15051e2..cb842c30e 100644 --- a/test/test_kb_feature_discovery.py +++ b/test/test_kb_feature_discovery.py @@ -43,10 +43,12 @@ def call(role, system, prompt, validate): engine = DiscoveryEngine(idx, seeds=[], owners=[], state={}, call=call, save=lambda: None, packet_chars=96000) assert engine.scan() - assert len(engine.state['tasks']) == 1 - task = next(iter(engine.state['tasks'].values())) - assert task['candidate_limit_reached'] - assert len(task['candidates']) == 24 + assert len(engine.state['tasks']) == 3 + tasks = list(engine.state['tasks'].values()) + assert all(task['candidate_limit_reached'] for task in tasks) + assert any(task.get('split_children') for task in tasks) + assert sum(bool(task.get('overflow_unknown')) for task in tasks) == 2 + assert all(len(task['candidates']) == 24 for task in tasks) def reply(role, data, archived=True): diff --git a/test/test_kb_feature_discovery_stage.py b/test/test_kb_feature_discovery_stage.py index fb0583205..bb6a6f1f7 100644 --- a/test/test_kb_feature_discovery_stage.py +++ b/test/test_kb_feature_discovery_stage.py @@ -266,6 +266,13 @@ def test_first_source_only_catalog_has_native_review_and_no_knowledge_rewrite(wo assert report["complete"] and report["done"] and report["feature_ids"] == ["engine-step"] assert report["historical_denominator"] == 0 and report["facet_denominator"] == 7 assert report["catalog_sha256"] == hashlib.sha256(files["adapters/toy/knowledge-coverage.yaml"].encode()).hexdigest() + assert report["scan_mode"] == "full" and report["scan_paths"] is None + assert report["unit_summary"]["associated"] == record.discovery["unit_resolution"]["associated"] + assert report["residual_summary"]["completed_rounds"] == 2 + bundle = report["evidence_bundles"]["engine-step"] + assert bundle["pin"] == record.pin and bundle["catalog_sha256"] == report["catalog_sha256"] + assert bundle["refs"] and bundle["unit_ids"] + assert all("text" not in ref for ref in bundle["refs"]) candidate = _full_report(record)["candidates"][0] assert candidate["status"] == "accepted" receipts = candidate["generator_receipts"] + [candidate["judge_receipt"]] diff --git a/test/test_kb_model_dispatch.py b/test/test_kb_model_dispatch.py new file mode 100644 index 000000000..7913caf68 --- /dev/null +++ b/test/test_kb_model_dispatch.py @@ -0,0 +1,157 @@ +"""Real shared slots across threads/processes; no model or network calls.""" + +import multiprocessing +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace + +import pytest + +from infermatrix_copilot.kb_service.model_dispatch import DispatchConfigError, SharedModelDispatch +from infermatrix_copilot.kb_service.models import ModelGateway, ModelRole, ModelUnavailable + + +def _process_call(directory, ready, active, maximum, mutex): + dispatch = SharedModelDispatch(directory, limit=3) + ready.wait(5) + with dispatch.slot(): + with mutex: + active.value += 1 + maximum.value = max(maximum.value, active.value) + time.sleep(.15) + with mutex: + active.value -= 1 + + +def _process_hold(directory, ready): + with SharedModelDispatch(directory, limit=1).slot(): + ready.set() + time.sleep(30) + + +def test_threads_share_thirteen_slots_across_distinct_limiter_instances(tmp_path): + active = maximum = 0 + mutex = threading.Lock() + saturated = threading.Event() + release = threading.Event() + def call(_): + nonlocal active, maximum + # Separate stage/gateway instances must still share the same pool. + with SharedModelDispatch(tmp_path).slot() as ticket: + assert ticket.limit == 13 and ticket.wait_seconds >= 0 + with mutex: + active += 1 + maximum = max(maximum, active) + if active == 13: + saturated.set() + assert release.wait(5) + with mutex: + active -= 1 + with ThreadPoolExecutor(max_workers=26) as pool: + futures = [pool.submit(call, n) for n in range(26)] + try: + assert saturated.wait(5) + finally: + release.set() + for future in futures: + future.result(timeout=5) + assert maximum == 13 and active == 0 + + +def test_processes_share_cap_and_release_every_completed_call(tmp_path): + ctx = multiprocessing.get_context("spawn") + ready, active, maximum, mutex = ctx.Event(), ctx.Value("i", 0), ctx.Value("i", 0), ctx.Lock() + processes = [ctx.Process(target=_process_call, args=(str(tmp_path), ready, active, maximum, mutex)) for _ in range(8)] + try: + for process in processes: + process.start() + ready.set() + for process in processes: + process.join(10) + assert process.exitcode == 0 + finally: + for process in processes: + if process.is_alive(): + process.terminate() + process.join(5) + assert 1 < maximum.value <= 3 and active.value == 0 + + +def test_crashed_process_releases_kernel_lock_without_stale_pid_cleanup(tmp_path): + ctx = multiprocessing.get_context("spawn") + ready = ctx.Event() + child = ctx.Process(target=_process_hold, args=(str(tmp_path), ready)) + try: + child.start() + assert ready.wait(5) + child.terminate() + child.join(5) + assert not child.is_alive() + started = time.monotonic() + with SharedModelDispatch(tmp_path, limit=1).slot(): + assert time.monotonic() - started < 1 + finally: + if child.is_alive(): + child.terminate() + child.join(5) + + +@pytest.mark.parametrize("limit", [0, 14, True, "13.0", "", "bad"]) +def test_cap_config_is_bounded_and_conflicting_namespace_refused(tmp_path, limit): + with pytest.raises(DispatchConfigError): + SharedModelDispatch(tmp_path, limit=limit) + + +def test_default_host_namespace_shared_across_batch_state_directories(tmp_path): + env = {"XDG_CACHE_HOME": str(tmp_path), "KB_INIT_GLOBAL_CONCURRENCY": "3"} + first = SharedModelDispatch(environ=env) + second = SharedModelDispatch(environ=env) + assert first.directory == second.directory == tmp_path / "infermatrix-copilot/kb-init-dispatch" + with pytest.raises(DispatchConfigError, match="conflicts"): + SharedModelDispatch(first.directory, limit=4) + assert SharedModelDispatch(environ={"KB_INIT_DISPATCH_DIR": str(tmp_path / "explicit")}).limit == 13 + + +def test_native_gateway_fallbacks_and_judges_use_same_slots_and_keep_recording(tmp_path): + active = maximum = 0 + mutex = threading.Lock() + records, providers = [], [] + def factory(provider): + class Transport: + def complete(self, **kwargs): + nonlocal active, maximum + with mutex: + active += 1 + maximum = max(maximum, active) + providers.append((provider, kwargs["role"])) + try: + time.sleep(.03) + if provider == "unavailable": + raise RuntimeError("scripted primary unavailable") + return SimpleNamespace(model=kwargs["model"], stop_reason="end_turn", usage={}, + blocks=[SimpleNamespace(text='{"supported":true}')]) + finally: + with mutex: + active -= 1 + return Transport() + fallback = ModelRole("generator", "fallback", "scripted") + generator = ModelRole("generator", "unavailable", "scripted", fallback=fallback) + judge = ModelRole("judge", "independent", "scripted") + # Different native gateways/roles all use the same host namespace. + gateways = [ModelGateway(SimpleNamespace(), transport_factory=factory, recorder=records.append) for _ in range(4)] + for gateway in gateways: + gateway.configure_dispatch(SharedModelDispatch(tmp_path, limit=2)) + with ThreadPoolExecutor(max_workers=8) as pool: + futures = [pool.submit(gateways[n % 4].call_json, generator if n % 2 else judge, + system="source-bound", prompt="scripted evidence") for n in range(8)] + assert all(future.result(timeout=5).data == {"supported": True} for future in futures) + assert maximum <= 2 and active == 0 + assert len(records) == 12 and sum(p == "fallback" for p, _ in providers) == 4 + assert all(row["system"] == "source-bound" and row["prompt"] == "scripted evidence" for row in records) + def reject(_): + raise ValueError("schema refused") + with pytest.raises(ModelUnavailable, match="schema"): + gateways[0].call_json(judge, system="source-bound", prompt="bad schema", validate=reject) + # Exception and schema rejection release slots for later independent calls. + assert gateways[0].call_json(judge, system="source-bound", prompt="resume").data["supported"] diff --git a/test/test_kb_portable_knowledge_store.py b/test/test_kb_portable_knowledge_store.py new file mode 100644 index 000000000..dba603602 --- /dev/null +++ b/test/test_kb_portable_knowledge_store.py @@ -0,0 +1,262 @@ +import hashlib +import json +from dataclasses import replace + +import pytest + +from infermatrix_copilot.knowledge_view import KnowledgeView, KnowledgeViewError +from infermatrix_copilot.kb_service.activate import verify_snapshot +from infermatrix_copilot.kb_service.knowledge_store import KnowledgeStore, KnowledgeStoreError, immutable_json, local_acceptance_receipt, validate_local_acceptance +from infermatrix_copilot.kb_service.repo_spec import RepoSpec, RepoSpecError, canonical_json, list_snapshot_repos, resolve_snapshot_repo + +S0 = "1" * 40 +S1 = "2" * 40 + + +def _tree(root, spec=None): + spec = spec or RepoSpec(repo_id="library", aliases=("lib",), source_pin=S0, + catalog_hash="a" * 64, policy_hash="b" * 64) + def page(title): + return f'---\ntitle: "{title}"\ncreated: 2026-10-06\nupdated: 2026-10-06\ntype: index\ntags: [general]\nsources: []\n---\n\n# {title}\n' + files = {"AGENTS.md": "Existing contribution format\n", "_format.yaml": "format_version: 2\n", + "_repositories.yaml": canonical_json({"schema_version": 1, "repos": {spec.repo_id: spec.to_dict()}}).decode(), + "repos/_index.md": page("Repositories"), "general/_index.md": page("General"), + "repos/library/_index.md": page("Library"), "repos/library/components/_index.md": page("Components"), + "repos/library/components/runtime/_index.md": page("Runtime"), + "repos/library/_routes.yaml": "schema_version: 1\nowners:\n- owner: runtime\n path: repos/library/components/runtime/_index.md\n signals: [api]\n scope_prefixes: [src/]\n"} + for rel, text in files.items(): + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + hashes = {rel: hashlib.sha256(text.encode()).hexdigest() for rel, text in files.items()} + return KnowledgeView(root, "fixed-snapshot", hashes), files + + +def test_served_registry_aliases_bound_only_to_current_verified_view(tmp_path): + view, _ = _tree(tmp_path / "knowledge") + binding = resolve_snapshot_repo(view, "lib") + assert binding.repo_id == "library" and binding.knowledge_slice == "repos/library" + assert binding.catalog_hash == "a" * 64 and binding.policy_hash == "b" * 64 + assert binding.source_pin == S0 and binding.owner_routes[0]["owner"] == "runtime" + assert list_snapshot_repos(view) == (binding,) + assert resolve_snapshot_repo(view, "unregistered") is None + assert resolve_snapshot_repo(view, "../library") is None + (view.root / "_repositories.yaml").write_text("tampered registry") + with pytest.raises(KnowledgeViewError): + resolve_snapshot_repo(view, "lib") + + +def test_old_snapshot_exact_slice_preserved_without_invented_pin_or_policy(tmp_path): + view, files = _tree(tmp_path / "knowledge") + (view.root / "_repositories.yaml").unlink() + del files["_repositories.yaml"] + view = KnowledgeView(view.root, "old", {k: hashlib.sha256(v.encode()).hexdigest() for k, v in files.items()}) + binding = resolve_snapshot_repo(view, "library") + assert binding.repo_id == "library" and binding.source_pin == "" and binding.catalog_hash == "" and binding.policy_hash == "" + assert resolve_snapshot_repo(view, "lib") is None + assert list_snapshot_repos(view) == (binding,) + + +def test_registry_ambiguous_alias_or_cross_slice_route_is_refused(tmp_path): + view, files = _tree(tmp_path / "knowledge") + registry = json.loads(files["_repositories.yaml"]) + registry["repos"]["other"] = RepoSpec(repo_id="other", aliases=("lib",)).to_dict() + (view.root / "_repositories.yaml").write_bytes(canonical_json(registry)) + plain = KnowledgeView(view.root, "unverified") + with pytest.raises(RepoSpecError, match="ambiguous"): + resolve_snapshot_repo(plain, "lib") + registry["repos"].pop("other") + (view.root / "_repositories.yaml").write_bytes(canonical_json(registry)) + (view.root / "repos/library/_routes.yaml").write_text("owners:\n- owner: other\n path: repos/other/_index.md\n") + with pytest.raises(RepoSpecError, match="crosses"): + resolve_snapshot_repo(plain, "library") + + +@pytest.mark.parametrize("mode", ["central", "in_repo"]) +def test_canonical_snapshot_same_format_with_separate_source_and_publication_commits(tmp_path, mode): + root = tmp_path / ("central/knowledge" if mode == "central" else "source/.infermatrix/knowledge") + _tree(root) + store = KnowledgeStore(root, "library", mode=mode) + manifest = store.snapshot(tmp_path / "accepted-snapshot", source_pin=S0, publication_head=S1) + assert manifest["source_pin"] == S0 and manifest["publication_head"] == S1 + assert manifest["source_pin"] != manifest["publication_head"] + assert manifest["knowledge_storage"] == mode and manifest["schema_version"] == 1 + verified = verify_snapshot(tmp_path / "accepted-snapshot") + assert resolve_snapshot_repo(verified, "library").source_pin == S0 + assert manifest["knowledge_format"] == 2 and manifest["tree_sha256"] == verified.tree_sha256 + with pytest.raises(KnowledgeStoreError): + store.snapshot(root / "self-update", source_pin=S0, publication_head=S1) + + +def test_mirror_is_verified_read_only_and_edited_or_unbound_copy_never_overwritten(tmp_path): + root = tmp_path / "canonical" + _tree(root) + store = KnowledgeStore(root, "library") + mirror = tmp_path / "mirror" + first = store.sync_mirror(mirror, source_pin=S0, publication_head=S1) + assert KnowledgeStore.verify_mirror(mirror) == first + automatic_readonly = KnowledgeStore(mirror / "knowledge", "library") + with pytest.raises(KnowledgeStoreError, match="read-only"): + automatic_readonly.write_files({"repos/library/_index.md": "edit"}) + page = mirror / "knowledge/repos/library/_index.md" + original = page.read_bytes() + page.write_bytes(b"local conflict") + with pytest.raises(KnowledgeStoreError): + store.sync_mirror(mirror, source_pin=S0, publication_head="3" * 40) + assert page.read_bytes() == b"local conflict" + page.write_bytes(original) + (mirror / "unbound.txt").write_text("user work") + with pytest.raises(KnowledgeStoreError, match="unbound"): + store.sync_mirror(mirror, source_pin=S0, publication_head=S1) + assert (mirror / "unbound.txt").read_text() == "user work" + + +def test_clean_mirror_update_keeps_canonical_authority_and_refuses_other_repository(tmp_path): + root = tmp_path / "canonical" + _tree(root) + store = KnowledgeStore(root, "library") + mirror = tmp_path / "mirror" + old = store.sync_mirror(mirror, source_pin=S0, publication_head=S1) + store.write_files({"repos/library/_index.md": "New reviewed index\n"}) + new = store.sync_mirror(mirror, source_pin=S0, publication_head="3" * 40) + assert old["snapshot"] != new["snapshot"] and new["source_pin"] == S0 + assert (mirror / "knowledge/repos/library/_index.md").read_text() == "New reviewed index\n" + with pytest.raises(KnowledgeStoreError, match="another repository"): + KnowledgeStore(root, "other").sync_mirror(mirror, source_pin=S0, publication_head=S1) + + +def test_all_destination_conflicts_checked_before_any_canonical_write(tmp_path): + root = tmp_path / "canonical" + _, files = _tree(root) + store = KnowledgeStore(root, "library") + rel = "repos/library/_index.md" + second = "repos/library/components/runtime/_index.md" + with pytest.raises(KnowledgeStoreError, match="changed"): + store.write_files({rel: "replacement", second: "replacement"}, expected_hashes={rel: hashlib.sha256(files[rel].encode()).hexdigest(), second: "0" * 64}) + assert (root / rel).read_text() == files[rel] + with pytest.raises(KnowledgeStoreError, match="crosses"): + store.write_files({"repos/other/_index.md": "wrong owner"}) + + +def test_local_acceptance_exact_files_heads_and_real_review_artifacts(tmp_path): + native = tmp_path / "genuine-review.json" + native.write_text('{"genuine_saved_review":true}\n') + files = {"knowledge/repos/library/_index.md": "Reviewed knowledge\n"} + kwargs = dict(repo_id="library", source_pin=S0, publication_head=S1, files=files) + receipt = local_acceptance_receipt(**kwargs, checks={"source_integrity": True, "native_review": True, "format": True}, accepted=True, review_receipts=(native,)) + assert receipt["source_pin"] == S0 and receipt["publication_head"] == S1 and receipt["forge"] == "none" + output = tmp_path / "local-acceptance.json" + h = immutable_json(output, receipt) + assert h == hashlib.sha256(output.read_bytes()).hexdigest() + validate_local_acceptance(receipt, **kwargs) + with pytest.raises(KnowledgeStoreError): + validate_local_acceptance(receipt, **{**kwargs, "files": {next(iter(files)): "changed"}}) + with pytest.raises(KnowledgeStoreError): + immutable_json(output, {**receipt, "publication_head": "3" * 40}) + native.write_text("changed actual review record") + with pytest.raises(KnowledgeStoreError): + validate_local_acceptance(receipt, **kwargs) + with pytest.raises(KnowledgeStoreError): + local_acceptance_receipt(**kwargs, checks={"native_review": None}, accepted=True) + with pytest.raises(KnowledgeStoreError): + local_acceptance_receipt(**kwargs, checks={"native_review": True}, accepted=False) + + +@pytest.mark.parametrize("tier", ["final", "foundation"]) +def test_mirror_preserves_actual_canonical_acceptance_tier_and_refuses_stale_scope(tmp_path, tier): + root = tmp_path / "canonical" + _tree(root) + store = KnowledgeStore(root, "library") + final = tier == "final" + acceptance = {"tier": tier, "init_complete": final, "source_pin": S0, "catalog_sha256": "a" * 64, + "checks": {"source": True, "native": True, "structural": True, + "semantic_depth": final, "retrieval": final}} + published = store.snapshot(tmp_path / "published", source_pin=S0, publication_head=S1, acceptance=acceptance) + assert published["acceptance"] == acceptance + assert verify_snapshot(tmp_path / "published").verified + copied = store.sync_mirror(tmp_path / "mirror", source_pin=S0, publication_head=S1) + assert copied["acceptance"] == acceptance and copied["canonical_snapshot"] == published["snapshot"] + assert copied["snapshot"] != published["snapshot"] + assert verify_snapshot(tmp_path / "mirror").verified + with pytest.raises(KnowledgeStoreError): + store.sync_mirror(tmp_path / "wrong-summary", source_pin=S0, publication_head=S1, + acceptance={**acceptance, "init_complete": not final}) + (root / "repos/library/_index.md").write_text("changed after acceptance") + with pytest.raises(KnowledgeStoreError, match="no longer binds"): + store.sync_mirror(tmp_path / "mirror", source_pin=S0, publication_head=S1) + + +def test_final_status_requires_passing_gates_and_matching_catalog(tmp_path): + root = tmp_path / "canonical" + _tree(root) + store = KnowledgeStore(root, "library") + incomplete = {"tier": "final", "init_complete": True, "source_pin": S0, "catalog_sha256": "a" * 64, + "checks": {"source": True, "native": True, "structural": True, + "semantic_depth": False, "retrieval": True}} + with pytest.raises(KnowledgeStoreError, match="required passing"): + store.record_acceptance(source_pin=S0, publication_head=S1, acceptance=incomplete) + with pytest.raises(KnowledgeStoreError, match="frozen served registry"): + store.record_acceptance(source_pin=S0, publication_head=S1, + acceptance={**incomplete, "catalog_sha256": "c" * 64, + "checks": {**incomplete["checks"], "semantic_depth": True}}) + + +def test_mirror_contains_only_selected_repository_and_shared_general(tmp_path): + root = tmp_path / "canonical" + _, files = _tree(root) + other = root / "repos/secret/_index.md" + other.parent.mkdir(parents=True) + other.write_text(files["repos/library/_index.md"].replace("Library", "Secret internal repo")) + other_summary = root / "_publication-secret.json" + other_summary.write_text('{"internal":"unrelated repo receipt"}\n') + (root / "repos/_index.md").write_text(files["repos/_index.md"] + "\n- [Secret](secret/_index.md)\n") + registry = json.loads(files["_repositories.yaml"]) + registry["repos"]["secret"] = RepoSpec(repo_id="secret", source_pin="3" * 40).to_dict() + (root / "_repositories.yaml").write_bytes(canonical_json(registry)) + store = KnowledgeStore(root, "library") + full = store.snapshot(tmp_path / "full", source_pin=S0, publication_head=S1) + mirror = store.sync_mirror(tmp_path / "mirror", source_pin=S0, publication_head=S1) + assert "repos/secret/_index.md" in full["files"] + assert not any("secret" in path for path in mirror["files"]) + assert mirror["canonical_snapshot"] == full["snapshot"] + assert mirror["snapshot"] != full["snapshot"] + served = verify_snapshot(tmp_path / "mirror") + assert list_snapshot_repos(served)[0].repo_id == "library" + assert len(list_snapshot_repos(served)) == 1 + assert resolve_snapshot_repo(served, "secret") is None + assert "Secret" not in served.read_text("repos/_index.md") + assert served.read_text("general/_index.md") == files["general/_index.md"] + assert "secret" not in served.read_text("_repositories.yaml") + + +def test_mirror_rejects_invalid_manifest_even_if_marker_rehashed(tmp_path): + root = tmp_path / "canonical" + _tree(root) + store = KnowledgeStore(root, "library") + mirror = tmp_path / "mirror" + store.sync_mirror(mirror, source_pin=S0, publication_head=S1) + path = mirror / "MANIFEST.json" + manifest = json.loads(path.read_text()) + manifest["tree_sha256"] = "0" * 64 + path.write_bytes(canonical_json(manifest)) + marker_path = mirror / "MIRROR.json" + marker = json.loads(marker_path.read_text()) + marker["manifest_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + marker_path.write_bytes(canonical_json(marker)) + with pytest.raises(KnowledgeStoreError, match="verified knowledge manifest"): + KnowledgeStore.verify_mirror(mirror) + + +def test_saved_acceptance_cannot_advertise_final_with_failed_gate(tmp_path): + root = tmp_path / "canonical" + _tree(root) + store = KnowledgeStore(root, "library") + acceptance = {"tier": "final", "init_complete": True, "source_pin": S0, "catalog_sha256": "a" * 64, + "checks": {"source": True, "native": True, "structural": True, + "semantic_depth": True, "retrieval": True}} + store.record_acceptance(source_pin=S0, publication_head=S1, acceptance=acceptance) + stored = json.loads(store.acceptance_path.read_text()) + stored["acceptance"]["checks"]["native"] = False + store.acceptance_path.write_bytes(canonical_json(stored)) + with pytest.raises(KnowledgeStoreError, match="required passing"): + store.sync_mirror(tmp_path / "mirror", source_pin=S0, publication_head=S1) diff --git a/test/test_kb_portable_repositories.py b/test/test_kb_portable_repositories.py new file mode 100644 index 000000000..cc4a90852 --- /dev/null +++ b/test/test_kb_portable_repositories.py @@ -0,0 +1,258 @@ +"""Real committed-tree and optional-forge tests; no model/network fixtures.""" + +import json +import subprocess +from pathlib import Path + +import pytest + +from infermatrix_copilot.kb_service.git_source import GitSource, GitHubForge, GitLabForge, forge_provider, public_locator +from infermatrix_copilot.kb_service.repo_spec import RepoRegistry, RepoSpec, RepoSpecError, acceptance_receipt, propose_repository, repository_inventory, canonical_json +from infermatrix_copilot.kb_service.sources import SourceError +from infermatrix_copilot.kb_service.init_support import UpstreamPin + + +def _git(root, *args): + result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False) + assert result.returncode == 0, result.stderr.decode() + return result.stdout.decode().strip() + + +def _repository(tmp_path, object_format="sha1"): + root = tmp_path / "source" + root.mkdir() + _git(root, "init", "-q", "--initial-branch=main", f"--object-format={object_format}") + _git(root, "config", "user.email", "fixture@example.invalid") + _git(root, "config", "user.name", "Fixture") + files = {"src/library.py": "def public(): return 1\n", "ui/index.ts": "export const api=1;\n", + "knowledge/runtime.py": "def supported_package(): return 1\n", "README.md": "Library usage\n", + "sdk/tests/deep/test_api.py": "assert True\n", "sdk/src/api.go": "package api\n", + "lib/custom.langx": "public capability x\n", "src/icon.png": b"\x89PNG\xff\0", + "vendor/copied.py": "foreign\n", "sdk/node_modules/pkg/index.js": "foreign\n", + ".infermatrix/knowledge/repos/example/generated.md": "own output\n"} + for path, body in files.items(): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(body.encode() if isinstance(body, str) else body) + for n in range(70): + target = root / f"lib/part{n:03d}.py" + target.write_text(f"def feature{n}(): return {n}\n") + _git(root, "add", ".") + _git(root, "commit", "-qm", "Initial implementation") + return root + + +@pytest.mark.parametrize("object_format,width", [("sha1", 40), ("sha256", 64)]) +def test_any_committed_git_tree_no_origin_dirty_files_and_mirror(tmp_path, object_format, width): + root = _repository(tmp_path, object_format) + source = GitSource(root) + pin = source.resolve() + assert len(pin) == width and source.object_format == object_format + (root / "src/library.py").write_text("dirty replacement") + assert source.read(pin, "src/library.py") == b"def public(): return 1\n" + assert source.origin() == "" + mirror = GitSource.acquire(root, tmp_path / "mirror.git") + assert mirror.resolve() == pin and mirror.git_dir == tmp_path / "mirror.git" + exported = mirror.export(pin, tmp_path / "export") + assert (exported / "src/library.py").read_bytes() == source.read(pin, "src/library.py") + assert not (exported / ".git").exists() + with pytest.raises(SourceError): + source.resolve("--help") + with pytest.raises(SourceError): + source.read(pin, "../outside") + + +def test_proposal_inventory_is_full_multilingual_nested_and_unknown_not_self_ingested(tmp_path): + source = GitSource(_repository(tmp_path)) + proposal = propose_repository(source, repo_id="library") + spec = RepoSpec.from_dict(proposal["spec"]) + inventory = proposal["inventory"] + assert proposal["accepted"] is False + assert "lib/part069.py" in inventory["production"] + assert "knowledge/runtime.py" in inventory["production"] + assert "sdk/tests/deep/test_api.py" in inventory["tests"] + assert "README.md" in inventory["docs"] + assert "src/icon.png" in inventory["assets"] and "src/icon.png" not in inventory["production"] + assert "lib/custom.langx" in inventory["unknown_language_paths"] + assert set(spec.languages) == {"python", "typescript", "go", "unknown"} + assert len(inventory["excluded"]) == 3 + restricted = RepoSpec.from_dict({**spec.to_dict(), "source_roots": ["src"]}) + outside = repository_inventory(source, restricted) + assert "sdk/src/api.go" in outside["scope_expansion_proposals"] + assert "sdk/src/api.go" not in outside["production"] + assert "README.md" in outside["docs"] and "sdk/tests/deep/test_api.py" in outside["tests"] + assert "README.md" not in outside["outside_scope"] + + +@pytest.mark.parametrize("object_format", ["sha1", "sha256"]) +def test_export_materializes_exact_blobs_despite_archive_attributes(tmp_path, object_format): + source = GitSource(_repository(tmp_path, object_format)) + (source.path / ".gitattributes").write_text("src/library.py export-ignore\nversion.txt export-subst\n") + (source.path / "version.txt").write_bytes(b"$Format:%H$\n\0binary\n") + _git(source.path, "add", ".") + _git(source.path, "commit", "-qm", "Archive attributes") + pin = source.head() + exported = source.export(pin, tmp_path / "export") + actual = {path.relative_to(exported).as_posix() for path in exported.rglob("*") if path.is_file()} + assert actual == set(source.files(pin)) + assert (exported / "src/library.py").read_bytes() == source.read(pin, "src/library.py") + assert (exported / "version.txt").read_bytes() == b"$Format:%H$\n\0binary\n" + bare = GitSource.acquire(source.path, tmp_path / "bare.git") + observer = UpstreamPin(bare.path, "repo://fixture", remote=str(source.path)) + actual = observer.export(observer.resolve(pin), tmp_path / "observer-export") + assert (actual / "src/library.py").read_bytes() == source.read(pin, "src/library.py") + assert (actual / "version.txt").read_bytes() == source.read(pin, "version.txt") + assert {p.relative_to(actual).as_posix() for p in actual.rglob("*") if p.is_file()} == set(source.files(pin)) + + +def test_marked_canonical_knowledge_is_reviewed_exclusion_not_package_name(tmp_path): + source = GitSource(_repository(tmp_path)) + previous = RepoSpec.from_dict(propose_repository(source, repo_id="library")["spec"]) + for path, text in {"shared-kb/AGENTS.md": "Canonical rules\n", "shared-kb/_format.yaml": "format_version: 2\n", + "shared-kb/repos/_index.md": "Repository index\n", "shared-kb/repos/library/new.md": "Generated knowledge\n"}.items(): + target = source.path / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + _git(source.path, "add", ".") + _git(source.path, "commit", "-qm", "Committed canonical KB") + proposal = propose_repository(source, repo_id="library") + assert "shared-kb/**" in proposal["spec"]["exclude"] + assert proposal["inventory"]["generated_knowledge_roots"] == ["shared-kb"] + assert "shared-kb/repos/library/new.md" in proposal["inventory"]["excluded"] + assert "knowledge/runtime.py" in proposal["inventory"]["production"] + # Existing accepted scope is reported unchanged until its new exclusion + # proposal is reviewed; recognizing a marker must not lower its denominator. + current = RepoSpec.from_dict({**previous.to_dict(), "source_pin": source.head()}) + inventory = repository_inventory(source, current) + assert inventory["generated_knowledge_exclusion_proposals"] == ["shared-kb/**"] + assert "shared-kb/repos/library/new.md" in inventory["docs"] + + +def test_generated_mirror_root_requires_actual_manifest_and_committed_byte_binding(tmp_path): + from infermatrix_copilot.knowledge_view import build_manifest + import hashlib + source = GitSource(_repository(tmp_path)) + root = source.path / "mirror-copy" + for path, body in {"knowledge/AGENTS.md": "Shared rules\n", "knowledge/_format.yaml": "format_version: 2\n", + "knowledge/repos/_index.md": "Repositories\n"}.items(): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body) + raw = canonical_json(build_manifest(root / "knowledge", "copied-source")) + (root / "MANIFEST.json").write_bytes(raw) + (root / "MIRROR.json").write_bytes(canonical_json({"readonly": True, "manifest_sha256": hashlib.sha256(raw).hexdigest()})) + _git(source.path, "add", ".") + _git(source.path, "commit", "-qm", "Bound generated mirror") + proposal = propose_repository(source, repo_id="library") + assert proposal["inventory"]["generated_knowledge_roots"] == ["mirror-copy"] + assert "mirror-copy/**" in proposal["spec"]["exclude"] + (root / "knowledge/repos/_index.md").write_text("Modified beyond mirror provenance\n") + _git(source.path, "add", ".") + _git(source.path, "commit", "-qm", "Unbound mirror bytes") + proposal = propose_repository(source, repo_id="library") + assert proposal["inventory"]["generated_knowledge_roots"] == ["mirror-copy/knowledge"] + assert "mirror-copy/**" not in proposal["spec"]["exclude"] + + +def test_stable_local_identity_and_sanitized_nested_gitlab_identity(tmp_path): + source = GitSource(_repository(tmp_path)) + local_id = propose_repository(source)["spec"]["repo_id"] + clone = GitSource.acquire(source.path, tmp_path / "moved.git") + assert propose_repository(clone)["spec"]["repo_id"] == local_id + _git(source.path, "remote", "add", "origin", "https://user:secret@git.example/group/nested/project.git?token=private") + proposal = propose_repository(source, forge_kind="gitlab") + assert proposal["spec"]["forge_project"] == "group/nested/project" + serialized = json.dumps(proposal) + assert "secret" not in serialized and "private" not in serialized and str(source.path) not in serialized + assert public_locator("git@git.example:group/nested/project.git") == "ssh://git.example/group/nested/project.git" + assert public_locator("C:\\work\\repo") == "" + + +def test_actual_commit_diffs_preserve_renames_as_deleted_and_added_paths(tmp_path): + source = GitSource(_repository(tmp_path)) + old = source.head() + assert source.parents(old) == () and source.changed_paths(old, old) == [] + _git(source.path, "mv", "src/library.py", "src/renamed.py") + (source.path / "new.ts").write_text("export const added=1;\n") + _git(source.path, "add", ".") + _git(source.path, "commit", "-qm", "Move and add") + new = source.head() + assert source.parents(new) == (old,) + assert source.changed_paths(old, new) == ["new.ts", "src/library.py", "src/renamed.py"] + _git(source.path, "remote", "add", "origin", "https://github.com/Owner/Repo.git") + https_id = propose_repository(source)["spec"]["repo_id"] + _git(source.path, "remote", "set-url", "origin", "git@github.com:Owner/Repo.git") + assert propose_repository(source)["spec"]["repo_id"] == https_id + + +@pytest.mark.parametrize("changes", [{"coverage_target": .84}, {"per_facet_gt": .89}, + {"coverage_target": True}, {"source_roots": ["../outside"]}, + {"source_locator": "https://user:secret@example/repo.git"}, + {"knowledge_slice": "repos/another"}]) +def test_every_repo_keeps_global_minimum_gates_and_safe_public_spec(changes): + with pytest.raises(RepoSpecError): + RepoSpec.from_dict({"repo_id": "repo", **changes}) + + +def test_registration_exact_scope_and_head_no_public_runtime_paths(tmp_path): + source = GitSource(_repository(tmp_path)) + spec = RepoSpec.from_dict(propose_repository(source, repo_id="library")["spec"]) + registry = RepoRegistry(tmp_path / "state") + receipt = acceptance_receipt(spec, source, source.head(), accepted=True) + registry.register(spec, receipt, source_path=source.path, knowledge_root=tmp_path / "canonical") + assert registry.resolve("library")["spec"] == spec.to_dict() + assert registry.get("library")["bindings"]["source_path"] == str(source.path) + public = canonical_json(registry.public_snapshot()) + assert str(source.path).encode() not in public and b"bindings" not in public + with pytest.raises(RepoSpecError): + registry.register(spec, {**receipt, "spec_sha256": "0" * 64}, source_path=source.path, knowledge_root=tmp_path / "canonical") + with pytest.raises(RepoSpecError): + acceptance_receipt(spec, source, source.head(), accepted=False) + + +def test_dependencies_are_public_versioned_links_without_runtime_bindings(): + link = {"repo_id": "sdk", "status": "confirmed", "source_pin": "1" * 40, + "catalog_hash": "a" * 64, "evidence": [{"path": "src/client.py", "start": 2, "end": 4}]} + spec = RepoSpec(repo_id="service", dependencies=(link,)) + assert spec.to_dict()["dependencies"] == [link] + assert RepoSpec(repo_id="service", dependencies=({"repo_id": "sdk", "status": "unknown"},)).dependencies + for invalid in ({**link, "source_path": "/runtime/private"}, {**link, "token": "runtime-only"}, + {**link, "source_pin": ""}, {**link, "catalog_hash": ""}, + {**link, "evidence": [{"path": "/runtime/private"}]}, + {**link, "evidence": [{"path": "src/a.py", "token": "private"}]}): + with pytest.raises(RepoSpecError): + RepoSpec(repo_id="service", dependencies=(invalid,)) + + +def test_optional_forges_share_git_independent_source_and_nested_project_encoding(): + calls = [] + def request(*args): + calls.append(args) + return {"number": 1} + gh = GitHubForge("owner/repo", token="runtime-only", request=request) + gh.open_review(title="T", body="B", head="topic", base="main") + gl = GitLabForge("group/nested/repo", host="https://git.example", token="runtime-only", request=request) + gl.open_review(title="T", body="B", head="topic", base="main") + gl.review(3) + assert calls[0][1] == "https://api.github.com/repos/owner/repo/pulls" + assert calls[1][1] == "https://git.example/api/v4/projects/group%2Fnested%2Frepo/merge_requests" + assert calls[1][2]["source_branch"] == "topic" + assert calls[2][1].endswith("/merge_requests/3") + assert "runtime-only" not in json.dumps(calls) + with pytest.raises(SourceError): + forge_provider("none").open_review(title="T", body="B", head="topic", base="main") + + +@pytest.mark.parametrize("kind", ["github", "gitlab"]) +def test_forge_acceptance_checks_exact_merged_head_without_claiming_ci(kind): + head, merge = "1" * 40, "2" * 40 + row = {"merged": True, "state": "merged", "head": {"sha": head}, "sha": head, "merge_commit_sha": merge} + provider = forge_provider(kind, "owner/repo", host="https://forge.example", request=lambda *args: row) + accepted = provider.acceptance(3, head) + assert accepted["review_head"] == head and accepted["publication_head"] == merge + assert accepted["ci_verified"] is False + with pytest.raises(SourceError): + provider.acceptance(3, "3" * 40) + row.update(merged=False, state="opened") + with pytest.raises(SourceError): + provider.acceptance(3, head) diff --git a/test/test_knowledge_context.py b/test/test_knowledge_context.py new file mode 100644 index 000000000..1935b6cae --- /dev/null +++ b/test/test_knowledge_context.py @@ -0,0 +1,204 @@ +import hashlib +import json +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from infermatrix_copilot.knowledge_context import ContextBudget, ContextError, KnowledgeContextService +from infermatrix_copilot.knowledge_view import KnowledgeView, KnowledgeViewError, build_manifest + + +PIN = "a" * 40 + + +def setup(tmp_path, body="Useful source-linked context. " * 200, *, dependencies=()): + root = tmp_path / "snapshot" / "knowledge" + root.mkdir(parents=True) + (root / "AGENTS.md").write_text("knowledge") + for name in ("r", "other"): + folder = root / "repos" / name + folder.mkdir(parents=True) + (folder / "doc.md").write_text( + f"---\ntitle: API\ntype: guide\nfeature: api\nsources: [acme/r@{PIN}:src/api.py]\n---\n\n" + body) + manifest = build_manifest(root, "snapshot-1") + (root.parent / "MANIFEST.json").write_text(json.dumps(manifest)) + view = KnowledgeView(root, "snapshot-1", manifest["files"]) + bindings = { + name: {"repo_id": name, "knowledge_slice": f"repos/{name}", "source_pin": PIN, + "catalog_hash": "b" * 64, "policy_hash": "c" * 64, + "dependencies": dependencies if name == "r" else ()} + for name in ("r", "other")} + service = KnowledgeContextService(view, tmp_path / "ledger.db", resolver=lambda view, selector: bindings.get(selector)) + return service, view, bindings + + +def open_session(service, initial=1000, maximum=4000, **kwargs): + return service.open_session("r", source_pin=PIN, review_id="review", budget=ContextBudget( + initial_tokens=initial, max_tokens=maximum, model_context_tokens=8000, + source_reserve_tokens=2000, output_reserve_tokens=1000, **kwargs))["session_id"] + + +def test_cumulative_budget_counts_headers_and_preserves_unicode(tmp_path): + service, _, _ = setup(tmp_path, "真实的源码解释。" * 1000) + session = open_session(service) + first = service.read(session, "repos/r/doc.md") + assert first["truncated"] + assert len(first["model_content"].encode()) <= 1000 + assert first["consumed_tokens"] <= 1000 + assert first["next_offset"] < 1000 + assert "content" not in first["documents"][0] + assert "真实" in first["model_content"] + assert first["actual_provider_usage"] == "unknown" + + +def test_same_request_is_idempotent_across_restart_and_expansion_continues(tmp_path): + service, view, bindings = setup(tmp_path) + session = open_session(service) + first = service.read(session, "repos/r/doc.md") + restarted = KnowledgeContextService(view, tmp_path / "ledger.db", resolver=lambda view, selector: bindings.get(selector)) + assert restarted.read(session, "repos/r/doc.md") == first + assert restarted.status(session)["consumed_tokens"] == first["consumed_tokens"] + restarted.expand(session, target_tokens=4000, reason="Read remaining API contract") + second = restarted.read(session, "repos/r/doc.md") + assert second["documents"][0]["continuation"] + assert second["consumed_tokens"] <= 4000 + assert second["consumed_tokens"] > first["consumed_tokens"] + + +def test_reserve_capacity_cannot_be_expanded_away(tmp_path): + service, _, _ = setup(tmp_path) + budget = ContextBudget(initial_tokens=24000, max_tokens=64000, model_context_tokens=50000, + source_reserve_tokens=32000, output_reserve_tokens=8192) + status = service.open_session("r", source_pin=PIN, review_id="small", budget=budget) + assert status["capacity_tokens"] == 9808 + assert status["active_tokens"] == 9808 + with pytest.raises(ContextError, match="capacity"): + service.expand(status["session_id"], target_tokens=10000, reason="Large PR") + + +def test_parallel_reads_cannot_overspend(tmp_path): + service, _, _ = setup(tmp_path) + session = open_session(service) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda n: service.read(session, "repos/r/doc.md", offset=n * 50), range(8))) + assert service.status(session)["consumed_tokens"] <= 1000 + assert any(result["status"] == "budget_exhausted" for result in results) + + +def test_cached_delivery_still_rejects_snapshot_tampering(tmp_path): + service, view, _ = setup(tmp_path) + session = open_session(service) + service.read(session, "repos/r/doc.md") + (view.root / "repos/r/doc.md").write_text("changed") + with pytest.raises(KnowledgeViewError): + service.read(session, "repos/r/doc.md") + + +def test_cached_packet_content_tampering_is_not_returned_unmetered(tmp_path): + import sqlite3 + service, _, _ = setup(tmp_path) + session = open_session(service) + service.read(session, "repos/r/doc.md") + with sqlite3.connect(tmp_path / "ledger.db") as db: + state = json.loads(db.execute("SELECT state FROM sessions WHERE id=?", (session,)).fetchone()[0]) + receipt = next(iter(state["requests"].values())) + receipt["result"]["model_content"] = "UNMETERED SUBSTITUTION " * 1000 + db.execute("UPDATE sessions SET state=? WHERE id=?", (json.dumps(state), session)) + with pytest.raises(ContextError, match="integrity"): + service.read(session, "repos/r/doc.md") + + +def test_crossrepo_requires_host_and_confirmed_exact_pin(tmp_path): + link = {"repo_id": "other", "status": "confirmed", "source_pin": PIN, "catalog_hash": "b" * 64} + service, view, bindings = setup(tmp_path, dependencies=(link,)) + session = open_session(service) + with pytest.raises(ContextError, match="authorized"): + service.read(session, "repos/other/doc.md", repository="other") + authorized = KnowledgeContextService(view, tmp_path / "allowed.db", resolver=lambda view, selector: bindings.get(selector), + allowed_repositories=("other",)) + session = open_session(authorized) + result = authorized.read(session, "repos/other/doc.md", repository="other") + assert result["documents"][0]["repo_id"] == "other" + assert authorized.status(session)["consumed_tokens"] > 0 + bindings["r"]["dependencies"] = ({**link, "source_pin": "d" * 40},) + with pytest.raises(ContextError, match="confirmed"): + authorized.read(session, "repos/other/doc.md", offset=4, repository="other") + + +def test_custom_tokenizer_identity_and_expansion_reason(tmp_path): + service, view, bindings = setup(tmp_path) + custom = KnowledgeContextService(view, tmp_path / "custom.db", resolver=lambda view, selector: bindings.get(selector), + tokenizer=lambda text: len(text), tokenizer_id="character-test-v1") + session = open_session(custom) + result = custom.read(session, "repos/r/doc.md") + assert result["token_accounting"] == "character-test-v1" + assert result["consumed_tokens"] == len(result["model_content"]) + with pytest.raises(ContextError, match="reason"): + custom.expand(session, target_tokens=2000, reason=" ") + with pytest.raises(ContextError, match="tokenizer"): + KnowledgeContextService(view, tmp_path / "custom.db", resolver=lambda view, selector: bindings.get(selector)).status(session) + + +def test_sdk_dynamic_snapshot_uses_budgeted_context_without_extra_excerpts(tmp_path, monkeypatch): + from infermatrix_copilot.kb_service.repo_spec import RepoSpec + from infermatrix_copilot.sdk.v1 import ChangedPath, DirectClient, DirectReviewRequest, RepositoryRef + import yaml + _, view, _ = setup(tmp_path, "A source-linked API contract. " * 1000) + (view.root / "repos/r/_index.md").write_text("Repository navigation") + guide = view.root / "general/review/guides/simplification-audit.md" + guide.parent.mkdir(parents=True) + guide.write_text("Read actual code before declaring a finding.") + spec = RepoSpec("r", aliases=("git.example/subgroup/r",), source_pin=PIN, + catalog_hash="b" * 64, policy_hash="c" * 64) + (view.root / "_repositories.yaml").write_text(yaml.safe_dump({"schema_version": 1, "repos": {"r": spec.to_dict()}})) + manifest = build_manifest(view.root, "snapshot-2") + (view.root.parent / "MANIFEST.json").write_text(json.dumps(manifest)) + monkeypatch.setenv("KNOWLEDGE_ROOT", str(view.root.parent)) + request = DirectReviewRequest("dynamic-review", RepositoryRef("git.example/subgroup/r"), + 0, PIN, "API", "API contract", (ChangedPath("src/api.py"),)) + client = DirectClient(knowledge_context_path=tmp_path / "sdk.db") + legacy = client.plan(request) + assert legacy.related_knowledge["content_chars"] <= 6000 + adaptive = client.plan_adaptive(request) + assert len(adaptive["model_content"].encode()) <= 24000 + assert len(adaptive["model_content"]) > legacy.related_knowledge["content_chars"] + assert all("excerpt" not in reference and "content" not in reference for reference in adaptive["documents"]) + assert adaptive["budget"]["identity"]["repository"]["catalog_hash"] == "b" * 64 + # A new SDK instance resumes from the pinned tree, not whichever tree is active later. + restarted = DirectClient(knowledge_context_path=tmp_path / "sdk.db") + assert restarted.expand_knowledge_context(adaptive["session_id"], target_tokens=64000, + reason="Read the remaining contract")["capacity_tokens"] == 64000 + + +def test_adaptive_git_sha256_pin_is_supported(tmp_path): + service, _, _ = setup(tmp_path) + status = service.open_session("r", source_pin="e" * 64, review_id="sha256-source") + assert status["identity"]["source_pin"] == "e" * 64 + + +def test_strict_docs_use_same_pinned_budget_and_manifest_checks(tmp_path, monkeypatch): + from types import SimpleNamespace + from infermatrix_copilot.engine.agent_runtime.knowledge import _repo_docs_tool + _, view, _ = setup(tmp_path) + (view.root / "repos/r/_index.md").write_text("navigation") + manifest = build_manifest(view.root, "strict-snapshot") + (view.root.parent / "MANIFEST.json").write_text(json.dumps(manifest)) + monkeypatch.setenv("KNOWLEDGE_ROOT", str(view.root.parent)) + run_dir = tmp_path / "strict-run" + run_dir.mkdir() + traces = [] + ctx = SimpleNamespace( + settings=SimpleNamespace(knowledge_dir=view.root, allowed_knowledge_repositories=(), knowledge_general_docs=[]), + state={"task_spec": {"repo": "r", "expected_head_sha": PIN}}, + params={"knowledge_context_profile": "adaptive", "knowledge_context_budget": {"initial_tokens": 1000, "max_tokens": 4000}}, + run_dir=run_dir, trace=SimpleNamespace(record=lambda event, **data: traces.append((event, data)))) + tools = _repo_docs_tool(ctx, None) + packet = json.loads(tools["doc_related"].handler(["src/api.py"])) + assert packet["consumed_tokens"] <= 1000 + assert traces[0][1]["model_content"] == packet["model_content"] + read = json.loads(tools["doc_read"].handler("repos/r/doc.md")) + assert read["consumed_tokens"] <= 1000 + assert "doc_context_expand" in tools + (view.root / "repos/r/doc.md").write_text("tampered") + with pytest.raises(KnowledgeViewError): + tools["doc_read"].handler("repos/r/doc.md") diff --git a/test/test_knowledge_retrieval.py b/test/test_knowledge_retrieval.py index d8ac5cfad..3e5c3c71b 100644 --- a/test/test_knowledge_retrieval.py +++ b/test/test_knowledge_retrieval.py @@ -124,10 +124,54 @@ def test_bounds_keep_large_depth_retrievable_and_budget_followup(tmp_path): _page(k, third.page, "Third context", third) related = KnowledgeDocs(k, "repos/r").related(["pkg/models/catalog.py"]) assert len(related["documents"]) == 2 and related["content_chars"] <= 6000 - assert related["documents"][0]["included_facets"] == ["api"] + assert related["documents"][0]["included_facets"] == [] + assert related["documents"][0]["available_facets"] == ["api"] + assert related["documents"][0]["partial"] assert all(d["more_available"] for d in related["documents"]) +def test_clipped_facet_never_claims_intact_injection_with_custom_meter(tmp_path): + from infermatrix_copilot.knowledge_context import KnowledgeContextService + from infermatrix_copilot.knowledge_view import KnowledgeView + + k = tmp_path / "knowledge" + _page(k, "repos/r/depth.md", _block(tmp_path, body="Long behavior. " * 6000)) + binding = {"repo_id": "r", "knowledge_slice": "repos/r", "source_pin": PIN, + "catalog_hash": "b" * 64, "policy_hash": "c" * 64} + service = KnowledgeContextService(KnowledgeView(k, "fixture"), tmp_path / "meter.db", + resolver=lambda view, selector: binding, tokenizer=lambda text: (len(text) + 3) // 4, + tokenizer_id="four-character-test-v1") + session = service.open_session("r", source_pin=PIN, review_id="large-facet")["session_id"] + result = service.related(session, ["pkg/models/catalog.py"]) + ref = result["documents"][0] + assert ref["partial"] + assert ref["available_facets"] == ["api"] + assert ref["included_facets"] == [] + assert ref["not_injected_facets"] == ["api"] + assert ref["included_facet_acceptance_modes"] == {} + service.expand(session, target_tokens=64000, reason="Read more related context") + assert service.related(session, ["pkg/models/catalog.py"])["status"] == "no_new_context" + + +def test_budget_clipping_removes_all_intact_facet_metadata(tmp_path): + from infermatrix_copilot.knowledge_context import ContextBudget, KnowledgeContextService + from infermatrix_copilot.knowledge_view import KnowledgeView + + k = tmp_path / "knowledge" + _page(k, "repos/r/depth.md", _block(tmp_path, body="Supported behavior. " * 150)) + binding = {"repo_id": "r", "knowledge_slice": "repos/r", "source_pin": PIN, + "catalog_hash": "b" * 64, "policy_hash": "c" * 64} + service = KnowledgeContextService(KnowledgeView(k, "fixture"), tmp_path / "meter.db", + resolver=lambda view, selector: binding) + session = service.open_session("r", source_pin=PIN, review_id="budget-clip", + budget=ContextBudget(initial_tokens=2000))["session_id"] + result = service.related(session, ["pkg/models/catalog.py"]) + ref = result["documents"][0] + assert ref["partial"] and result["truncated"] + assert ref["included_facets"] == [] and ref["not_injected_facets"] == ["api"] + assert ref["included_facet_basis"] == ref["included_facet_acceptance_modes"] == {} + + def test_snapshot_verifier_runs_before_read_even_for_excluded_cards(tmp_path): k = tmp_path / "knowledge" path = _page(k, "repos/r/depth.md", _block(tmp_path)) diff --git a/test/test_portable_commands.py b/test/test_portable_commands.py new file mode 100644 index 000000000..3e35c0737 --- /dev/null +++ b/test/test_portable_commands.py @@ -0,0 +1,85 @@ +"""Portable CLI adapters and incremental batches on real committed sources.""" +import json +import subprocess +from types import SimpleNamespace + +import pytest + +from infermatrix_copilot.kb_service.git_source import GitSource +from infermatrix_copilot.kb_service.init_support import InitRecord +from infermatrix_copilot.kb_service.portable_commands import incremental_update, run_portable_init +from infermatrix_copilot.kb_service.repo_spec import RepoRegistry, RepoSpec, acceptance_receipt, propose_repository + + +def _git(root, *args): + return subprocess.check_output(["git", "-C", str(root), *args], stderr=subprocess.PIPE).decode().strip() + + +def _registered(tmp_path): + root = tmp_path / "source" + root.mkdir() + _git(root, "init", "-q", "-b", "main") + _git(root, "config", "user.name", "Fixture") + _git(root, "config", "user.email", "fixture@example.invalid") + (root / "api.py").write_text("def api(): return 1\n") + _git(root, "add", ".") + _git(root, "commit", "-qm", "API") + source = GitSource(root) + spec = RepoSpec.from_dict(propose_repository(source, repo_id="api-library")["spec"]) + state = tmp_path / "state" + RepoRegistry(state).register(spec, acceptance_receipt(spec, source, source.head(), accepted=True), + source_path=root, knowledge_root=tmp_path / "canonical/knowledge") + return state, source, spec + + +def test_knowledge_only_source_commit_does_not_start_discovery(tmp_path): + state, source, spec = _registered(tmp_path) + generated = source.path / ".infermatrix/knowledge/repos/api-library" + generated.mkdir(parents=True) + (generated / "_index.md").write_text("Generated knowledge\n") + _git(source.path, "add", ".") + _git(source.path, "commit", "-qm", "Knowledge mirror only") + report = incremental_update(state, spec.repo_id, "HEAD", apply=True) + assert report["implementation_unchanged"] is True + assert report["effective_changed_paths"] == [] + assert not (state / "batches").exists() + assert RepoRegistry(state).get(spec.repo_id)["spec"]["source_pin"] == spec.source_pin + + +def test_incremental_batch_pins_new_source_and_reports_deleted_paths(tmp_path): + state, source, spec = _registered(tmp_path) + (source.path / "api.py").unlink() + (source.path / "new.langx").write_text("public function new_contract() {}\n") + _git(source.path, "add", "-A") + _git(source.path, "commit", "-qm", "New public contract") + report = incremental_update(state, spec.repo_id, "HEAD", apply=True) + assert report["changed_paths"] == ["api.py", "new.langx"] + assert report["scan_paths"] == ["new.langx"] + assert report["init_complete"] is False + batch = state / "batches" / spec.repo_id / source.head() + assert RepoRegistry(batch).get(spec.repo_id)["spec"]["source_pin"] == source.head() + assert RepoRegistry(state).get(spec.repo_id)["spec"]["source_pin"] == spec.source_pin + assert json.loads((batch / "incremental.json").read_text())["from_pin"] == spec.source_pin + + +@pytest.mark.parametrize("explicit,expected", [(False, "lightweight"), (True, "strict")]) +def test_portable_defaults_do_not_override_explicit_strict_mode(tmp_path, monkeypatch, explicit, expected): + import infermatrix_copilot.kb_service.portable_commands as commands + import infermatrix_copilot.kb_service.init_stages as stages + pin = "a" * 40 + runtime = SimpleNamespace(generator=SimpleNamespace(provider="zcode"), + portable_spec=SimpleNamespace(source_pin=pin)) + monkeypatch.setattr(commands, "portable_runtime", lambda *a: (runtime, SimpleNamespace(repo="library"))) + captured = {} + def run(*a, **kwargs): + captured.update(kwargs) + return InitRecord("knowledge-deepen", "library", pin=pin, status="dry_run") + monkeypatch.setattr(stages, "run_stage", run) + args = SimpleNamespace(repo="library", stage="knowledge-deepen", suggest_seeds=False, + pin=None, budget_usd=None, acceptance_mode="strict", acceptance_mode_explicit=explicit, + foundation_mode="strict", from_existing=False, retry_unfinished=False, + subscription_generator=False, unlimited_subscription=False, pr_count=None) + assert run_portable_init(args, tmp_path) == 0 + assert captured["acceptance_mode"] == expected + assert captured["unlimited_subscription"] is True + assert captured["foundation_mode"] == "partial" diff --git a/test/test_portable_init_flow.py b/test/test_portable_init_flow.py new file mode 100644 index 000000000..d28911c79 --- /dev/null +++ b/test/test_portable_init_flow.py @@ -0,0 +1,400 @@ +"""Portable initialization on real Git trees and real offline native receipts. + +The transports are scripted; inventory, committed source export, stage gates, +validators, trace archives and local acceptance use their production paths. +""" +from __future__ import annotations + +import hashlib +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from infermatrix_copilot.kb_service.feature_discovery_index import build_for_stage +from infermatrix_copilot.kb_service.git_source import GitSource +from infermatrix_copilot.kb_service.gate import JUDGE_SYSTEM +from infermatrix_copilot.kb_service.depth_inputs import SYSTEM_DEPTH_LIGHTWEIGHT +from infermatrix_copilot.kb_service.init_knowledge_inputs import SYSTEM_KNOWLEDGE +from infermatrix_copilot.kb_service.init_feature_discovery import SYSTEM_CONSOLIDATE, SYSTEM_DISCOVER, SYSTEM_REVIEW +from infermatrix_copilot.kb_service.init_stages import SYSTEM_MAP, SYSTEM_RULES, run_stage +from infermatrix_copilot.kb_service.init_support import InitError, InitRecord +from infermatrix_copilot.kb_service.models import ModelGateway +from infermatrix_copilot.kb_service.portable_init import accept_stage, bootstrap_workspace, portable_runtime, stage_receipt +from infermatrix_copilot.kb_service.portable_commands import publish_portable +from infermatrix_copilot.kb_service.portable_publication import check_publication +from infermatrix_copilot.kb_service.knowledge_store import KnowledgeStoreError +from infermatrix_copilot.kb_service.knowledge_coverage import inventory, load_policy +from infermatrix_copilot.kb_service.activate import verify_snapshot +from infermatrix_copilot.kb_service.repo_spec import RepoRegistry, RepoSpec, acceptance_receipt, propose_repository, repository_inventory +from infermatrix_copilot.kb_service.runtime import trace_recorder +from infermatrix_copilot.llm import Block, Reply +from infermatrix_copilot.trace_store import TraceStore + + +def _git(root, *args): + result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False) + assert result.returncode == 0, result.stderr.decode() + return result.stdout.decode().strip() + + +class PortableNativeScript: + subscription_billing = True + supports_native_events = True + stops_at_spend = True + + def __init__(self): + self.calls = [] + + def complete(self, *, system, messages, model, role, native_event_sink=None, **kwargs): + prompt = messages[0]["content"] + payload = json.loads(prompt.split("\n", 1)[1].rsplit("", 1)[0]) + self.calls.append({"system": system, "role": role, "payload": payload, "model": model}) + if system == SYSTEM_MAP: + page = next(p for p in payload["offered_pages"] if p.endswith("/rules.md")) + data = {"title": "Portable library", "rules_title": "Library rules", "contents_heading": "Pages", + "index_intro": "Public library and web interface entry points.", + "architecture_md": "The library exports a value; the web interface exposes a ready contract.", + "owners": [{"owner": "core", "title": "Core", "page": page, + "signals": ["public", "ready"], "scope_prefixes": ["src/", "ui/", "lib/"]}], + "general_links": []} + elif system == SYSTEM_RULES: + data = {"rules": []} + elif system == SYSTEM_DISCOVER: + rows = [] + if payload["round"] != "doc": + for path, line, fid, description in ( + ("src/library.py", 1, "public-library", "Return the public library value."), + ("ui/ready.ts", 530, "ready-contract", "Expose the frontend ready contract.")): + if any(f["path"] == path and f["start"] <= line <= f["end"] for f in payload.get("files", [])): + rows.append({"id": fid, "title": fid.replace("-", " ").title(), "owner": "core", + "description": description, "relation": "new", "related_id": "", "aliases": [], + "evidence": [{"path": path, "start": line, "end": line}]}) + data = {"candidates": rows} + elif system in (SYSTEM_REVIEW, SYSTEM_CONSOLIDATE): + data = {"decisions": {row["id"]: {"supported": "yes", "relation": "new", "related_id": "", + "reason": "The cited public implementation supports this distinct contract."} + for row in payload["candidates"]}} + elif system.startswith(SYSTEM_KNOWLEDGE): + sections = [] + if "api" in payload["facets"]: + if any(f["path"] == "src/library.py" for f in payload["files"]): + ref = {"path": "src/library.py", "start": 1, "end": 1} + body = ("The public_value entry takes no arguments and returns the integer 7. " + "This exact function returns its constant value without reading any caller argument.") + else: + ref = {"path": "ui/ready.ts", "start": 530, "end": 530} + body = ("The exported ready entry takes no arguments and returns true as a boolean. " + "The visible return branch is unconditional within this exact function.") + sections = [{"facet": "api", "title": "Public return contract", "body": body, + "interpretation": "fact", "evidence": [ref]}] + data = {"title": "Supported API context", "sections": sections} + elif system == JUDGE_SYSTEM: + data = {"dimensions": {dimension: "yes" for dimension in payload["dimensions_to_answer"]}, + "reasons": {"faithful": "The exact shown branch establishes this narrow return contract."}} + elif system == SYSTEM_DEPTH_LIGHTWEIGHT: + data = {"title": "Unresolved depth", "sections": [], + "unknown_facets": [{"facet": facet, "reason": "The fixture intentionally leaves this depth explanation unsupported."} + for facet in payload["facets"]]} + else: + raise AssertionError(f"unexpected model call: {system[:100]}") + if native_event_sink: + native_event_sink({"type": "native.session.started", "payload": {"session_id": role + "-portable-fixture"}}) + native_event_sink({"type": "native.response", "payload": {"model": model}}) + return Reply(blocks=[Block(type="text", text=json.dumps(data))], stop_reason="end_turn", + usage={"input_tokens": 50, "output_tokens": 25}, model=model) + + +@pytest.fixture +def portable_world(tmp_path, monkeypatch, request): + monkeypatch.delenv("KB_GENERATOR", raising=False) + monkeypatch.delenv("KB_JUDGE", raising=False) + # Archive-only publication replay needs no credentials or ambient .env; + # keep fixture failure diagnostics independent of host Settings secrets. + monkeypatch.setattr("infermatrix_copilot.config.Settings", lambda: SimpleNamespace()) + source_root = tmp_path / "local-library" + source_root.mkdir() + overrides = dict(getattr(request, "param", {})) + object_format = overrides.pop("git_object_format", "sha1") + _git(source_root, "init", "-q", "-b", "main", f"--object-format={object_format}") + _git(source_root, "config", "user.email", "fixture@example.invalid") + _git(source_root, "config", "user.name", "Portable fixture") + ts = "\n".join(["// filler"] * 529 + ["export function ready(): boolean { return true; }"] + ["// tail"] * 90) + "\n" + source_files = {"src/__init__.py": "", "src/library.py": "def public_value(): return 7\n", "ui/ready.ts": ts, + "lib/contract.langx": "public function unknown_language() { return caller_value; }\n", + "sdk/tests/deep/test_library.py": "from src.library import public_value\nassert public_value() == 7\n", + "README.md": "# Portable library\n\nUse the public library value and frontend ready contract.\n", + "vendor/foreign.py": "def excluded(): return 0\n"} + for rel, text in source_files.items(): + target = source_root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + _git(source_root, "add", ".") + _git(source_root, "commit", "-qm", "Fixture public contracts") + source = GitSource(source_root) + proposal = propose_repository(source, repo_id="portable-library") + spec = RepoSpec.from_dict({**proposal["spec"], **overrides}) + state = tmp_path / "state" + registry = RepoRegistry(state) + registry.register(spec, acceptance_receipt(spec, source, source.head(), accepted=True), + source_path=source_root, knowledge_root=tmp_path / "canonical" / "knowledge") + script = PortableNativeScript() + traces = TraceStore(state / "init" / "traces") + gateway = ModelGateway(None, transport_factory=lambda provider: script, recorder=trace_recorder(traces)) + gateway.configure_zcode_pacing = lambda pacer: None + rt, lifecycle = portable_runtime(None, state, spec.repo_id, gateway=gateway) + rt.environ = {} + return {"source": source, "spec": spec, "state": state, "registry": registry, + "rt": rt, "lifecycle": lifecycle, "script": script, "traces": traces} + + +def _skeleton(world): + result = run_stage(world["rt"], world["lifecycle"], "skeleton", dry_run=True, + pin=world["spec"].source_pin, subscription_generator=True) + assert result.status == "dry_run", result.problems + receipt = stage_receipt(world["state"], world["spec"].repo_id, "skeleton", reviewer="Fixture reviewer") + return accept_stage(world["state"], world["spec"].repo_id, "skeleton", receipt) + + +def _discovery(world): + return run_stage(world["rt"], world["lifecycle"], "feature-discovery", dry_run=True, + pin=world["spec"].source_pin, unlimited_subscription=True) + + +def test_portable_bootstrap_full_inventory_preserves_nested_tests_and_unknown_language(portable_world, tmp_path): + world = portable_world + root = bootstrap_workspace(world["state"], world["registry"].get(world["spec"].repo_id)) + assert root == world["rt"].knowledge.path + manifest = yaml.safe_load((root / "adapters" / world["spec"].repo_id / "manifest.yaml").read_text()) + inv = repository_inventory(world["source"], world["spec"]) + assert set(inv["production"]) == {"src/__init__.py", "src/library.py", "ui/ready.ts", "lib/contract.langx"} + assert inv["tests"] == ["sdk/tests/deep/test_library.py"] + assert ".langx" in manifest["portable_scope"]["suffixes"] + assert manifest["knowledge_lifecycle"]["enabled"] is False + # Dirty source changes never contaminate the committed source export. + (world["source"].path / "src/library.py").write_text("dirty source\n") + tree = world["source"].export(world["spec"].source_pin, tmp_path / "committed-source") + stage = type("IndexStage", (), {"manifest": manifest, "lifecycle": world["lifecycle"], + "record": InitRecord("feature-discovery", world["spec"].repo_id, pin=world["spec"].source_pin), + "rt": world["rt"], "_base_sha": world["rt"].knowledge.main_sha()})() + index = build_for_stage(tree, stage) + assert index.files["src/library.py"]["text"] == "def public_value(): return 7\n" + assert set(index.production) == set(inv["production"]) - {"src/__init__.py"} + assert index.entries["src/__init__.py"]["classification"] == "empty_package_marker" + assert "sdk/tests/deep/test_library.py" in index.files + assert index.files["sdk/tests/deep/test_library.py"]["kind"] == "test" + assert "vendor/foreign.py" not in index.files + + +@pytest.mark.parametrize("portable_world", [{}, {"git_object_format": "sha256"}], indirect=True) +def test_portable_accepted_skeleton_discovery_preserves_native_hashes_and_long_ts_evidence(portable_world): + world = portable_world + skeleton = _skeleton(world) + assert skeleton.status == "accepted" and not skeleton.pr.get("number") + result = _discovery(world) + assert result.status == "dry_run", result.problems + assert result.pr["checked_files_sha256"] + receipt = stage_receipt(world["state"], world["spec"].repo_id, "feature-discovery", reviewer="Fixture reviewer") + accepted = accept_stage(world["state"], world["spec"].repo_id, "feature-discovery", receipt) + assert accepted.status == "accepted" + report_text = world["rt"].knowledge.show(accepted.pr["local_acceptance"]["head_sha"], accepted.discovery["report_path"]) + assert hashlib.sha256(report_text.encode()).hexdigest() == accepted.discovery["report_sha256"] + report = json.loads(report_text) + assert report["scan_mode"] == "full" + assert report["scan_summary"]["production_files"] == 3 + # An empty package marker is present in the reviewed scope and retained in + # the shared inventory, but contributes no implementation to core coverage. + catalog_path = f"adapters/{world['spec'].repo_id}/knowledge-coverage.yaml" + catalog_text = world["rt"].knowledge.show(accepted.pr["local_acceptance"]["head_sha"], catalog_path) + policy = load_policy(catalog_text, world["lifecycle"].knowledge_dir) + tree = world["source"].export(world["spec"].source_pin, world["source"].path.parent / "accepted-catalog-source") + assert set(inventory(tree, policy)) == {"src/library.py", "ui/ready.ts", "lib/contract.langx"} + assert set(report["evidence_bundles"]) == {"public-library", "ready-contract"} + assert any(ref["path"] == "ui/ready.ts" and ref["start"] == 530 + for ref in report["evidence_bundles"]["ready-contract"]["refs"]) + for bundle in report["evidence_bundles"].values(): + assert bundle["pin"] == world["spec"].source_pin + assert bundle["receipts"] + for receipt in bundle["receipts"]: + archived = world["traces"].get(receipt["trace_id"]) + assert archived["result"]["native_attempt_id"] + assert archived["model"]["provider"] in ("zcode", "codex") + # Both validators ran against the actual preview: no CI/gate function is mocked. + assert result.problems == [] + assert {c["role"] for c in world["script"].calls} == {"generator", "judge"} + assert not list(world["state"].rglob("kb.db")) + modules = run_stage(world["rt"], world["lifecycle"], "modules", dry_run=True, + pin=world["spec"].source_pin, unlimited_subscription=True) + assert modules.status in ("dry_run", "empty"), modules.problems + assert modules.discovery["catalog_binding"]["catalog_sha256"] == report["catalog_sha256"] + assert modules.coverage["modules"]["after"] == 1.0 + assert modules.coverage["unrouted"] == [] + + +@pytest.mark.parametrize("stage", ["skeleton", "feature-discovery"]) +def test_portable_direct_stage_rejects_new_source_pin_before_model_calls(portable_world, stage): + world = portable_world + (world["source"].path / "src/library.py").write_text("def public_value(): return 8\n", encoding="utf-8") + _git(world["source"].path, "add", "src/library.py") + _git(world["source"].path, "commit", "-qm", "Change source after approved proposal") + changed_pin = world["source"].head() + assert changed_pin != world["spec"].source_pin + calls = len(world["script"].calls) + with pytest.raises(InitError, match="portable source pin differs from the accepted batch"): + run_stage(world["rt"], world["lifecycle"], stage, dry_run=True, + pin=changed_pin, subscription_generator=True, + unlimited_subscription=stage == "feature-discovery") + assert len(world["script"].calls) == calls + assert InitRecord.load(world["state"], world["spec"].repo_id, stage) is None + + +def test_portable_unaccepted_catalog_blocks_modules_before_model_calls(portable_world): + world = portable_world + _skeleton(world) + catalog = _discovery(world) + assert catalog.status == "dry_run", catalog.problems + calls = len(world["script"].calls) + result = run_stage(world["rt"], world["lifecycle"], "modules", dry_run=True, + pin=world["spec"].source_pin, unlimited_subscription=True) + assert result.status == "blocked" + assert any("discovery" in message and ("accept" in message or "catalog" in message) for message in result.problems) + assert len(world["script"].calls) == calls + + +def test_portable_acceptance_refuses_modified_preview(portable_world): + world = portable_world + _skeleton(world) + result = _discovery(world) + assert result.status == "dry_run", result.problems + report = Path(result.pr["dry_run_dir"]) / "tree" / result.discovery["report_path"] + report.write_text(report.read_text() + "\n") + with pytest.raises(InitError, match="preview files changed"): + stage_receipt(world["state"], world["spec"].repo_id, "feature-discovery", reviewer="Fixture reviewer") + + +@pytest.mark.parametrize("portable_world", [{"per_facet_gt": 0.95}], indirect=True) +def test_portable_new_catalog_retains_reviewed_stricter_facet_target(portable_world): + world = portable_world + _skeleton(world) + result = _discovery(world) + assert result.status == "dry_run", result.problems + path = Path(result.pr["dry_run_dir"]) / "tree" / "adapters" / world["spec"].repo_id / "knowledge-coverage.yaml" + policy = yaml.safe_load(path.read_text()) + assert policy["semantic_depth"]["per_facet_gt"] == 0.95 + + +@pytest.mark.parametrize("mismatch", ["scope", "targets", "missing_depth"]) +def test_portable_existing_catalog_migration_cannot_shrink_scope_or_relax_targets(portable_world, mismatch): + world = portable_world + _skeleton(world) + policy = {"schema_version": 1, "required": True, + "core": {"roots": ["."], "suffixes": [".py", ".ts", ".langx"], + "exclude": list(world["spec"].exclude) + list(world["spec"].test_globs), "target": 0.85}, + "semantic_depth": {"per_facet_gt": 0.90, "acceptance_mode": "lightweight"}, + "features": [{"id": "existing-library", "title": "Existing library", "owner": "core", "docs": [], + "source_globs": ["src/library.py"], "entry_points": ["src/library.py"], + "page": "repos/portable-library/components/core/feature-existing-library.md"}]} + if mismatch == "scope": + policy["core"]["suffixes"] = [".py"] + elif mismatch == "targets": + policy["semantic_depth"]["per_facet_gt"] = 0.89 + else: + policy.pop("semantic_depth") + root = world["rt"].knowledge.path + path = root / "adapters" / world["spec"].repo_id / "knowledge-coverage.yaml" + path.write_text(yaml.safe_dump(policy, sort_keys=False)) + _git(root, "add", ".") + _git(root, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "Import historical catalog for migration audit") + calls = len(world["script"].calls) + result = _discovery(world) + assert result.status == "blocked" + assert any("scope/policy migration" in message for message in result.problems) + assert result.discovery["scope_migration"]["status"] == "reviewed_proposal_required" + assert len(world["script"].calls) == calls + + +@pytest.mark.parametrize("portable_world", [{}, {"git_object_format": "sha256"}], indirect=True) +def test_portable_foundation_unknowns_continue_to_depth_without_final_acceptance(portable_world): + world = portable_world + _skeleton(world) + discovery = _discovery(world) + assert discovery.status == "dry_run", discovery.problems + receipt = stage_receipt(world["state"], world["spec"].repo_id, "feature-discovery", reviewer="Fixture reviewer") + accept_stage(world["state"], world["spec"].repo_id, "feature-discovery", receipt) + modules = run_stage(world["rt"], world["lifecycle"], "modules", dry_run=True, + pin=world["spec"].source_pin, unlimited_subscription=True) + assert modules.status in ("empty", "dry_run"), modules.problems + if modules.status == "dry_run": + receipt = stage_receipt(world["state"], world["spec"].repo_id, "modules", reviewer="Fixture reviewer") + accept_stage(world["state"], world["spec"].repo_id, "modules", receipt) + foundation = run_stage(world["rt"], world["lifecycle"], "knowledge", dry_run=True, + pin=world["spec"].source_pin, unlimited_subscription=True) + assert foundation.status == "dry_run", foundation.problems + summary = foundation.coverage["foundation"] + assert summary["mode"] == "partial" and summary["tier"] == "foundation" + assert summary["init_complete"] is False and summary["foundation_targets_met"] is False + assert set(summary["unknown_features"]) == {"public-library", "ready-contract"} + assert all("validation" in facets and "api" not in facets for facets in summary["unknown_features"].values()) + receipt = stage_receipt(world["state"], world["spec"].repo_id, "knowledge", reviewer="Fixture reviewer") + accepted = accept_stage(world["state"], world["spec"].repo_id, "knowledge", receipt) + assert accepted.status == "accepted" and accepted.coverage["foundation"] == summary + snapshot = world["state"].parent / "foundation-snapshot" + calls = len(world["script"].calls) + manifest = publish_portable(world["state"], world["spec"].repo_id, snapshot, partial=True) + assert len(world["script"].calls) == calls + assert manifest["acceptance"]["tier"] == "foundation" + assert manifest["acceptance"]["init_complete"] is False + view = verify_snapshot(snapshot) + with pytest.raises(KnowledgeStoreError, match="foundation publication is incomplete"): + check_publication(view) + assert check_publication(view, allow_partial=True) == { + "tiers": {world["spec"].repo_id: "foundation"}, "init_complete": False} + canonical = Path(world["registry"].get(world["spec"].repo_id)["bindings"]["knowledge_root"]) + before = {path.relative_to(canonical).as_posix(): path.read_bytes() + for path in canonical.rglob("*") if path.is_file()} + occupied = world["state"].parent / "occupied-snapshot" + occupied.mkdir() + (occupied / "keep.txt").write_bytes(b"preexisting destination") + with pytest.raises(InitError, match="destination exists"): + publish_portable(world["state"], world["spec"].repo_id, occupied, partial=True) + assert {path.relative_to(canonical).as_posix(): path.read_bytes() + for path in canonical.rglob("*") if path.is_file()} == before + assert (occupied / "keep.txt").read_bytes() == b"preexisting destination" + depth = run_stage(world["rt"], world["lifecycle"], "knowledge-deepen", dry_run=True, + pin=world["spec"].source_pin, unlimited_subscription=True, + acceptance_mode="lightweight", foundation_mode="partial") + assert any(call["system"] == SYSTEM_DEPTH_LIGHTWEIGHT for call in world["script"].calls) + assert depth.depth["foundation"]["local_head_sha"] == accepted.pr["local_acceptance"]["head_sha"] + assert depth.depth["target_met"] is False and depth.depth["done"] is False + assert depth.coverage["semantic_depth"]["total_facets"] == 14 + assert depth.coverage["semantic_depth"]["recognized_facets"] == 0 + assert depth.status in ("blocked", "partial") + with pytest.raises(InitError, match="finished preview"): + stage_receipt(world["state"], world["spec"].repo_id, "knowledge-deepen", reviewer="Fixture reviewer") + with pytest.raises(InitError, match="explicitly accepted knowledge preview"): + publish_portable(world["state"], world["spec"].repo_id, + world["state"].parent / "forbidden-final-snapshot") + assert {path.relative_to(canonical).as_posix(): path.read_bytes() + for path in canonical.rglob("*") if path.is_file()} == before + + +@pytest.mark.parametrize("repository", ["repo://library", "owner/library", "https://git.example/group/nested/library"]) +def test_sha256_source_links_and_native_depth_prose_replay_are_exact(tmp_path, repository): + from infermatrix_copilot.kb_service.knowledge_depth import render_block + from infermatrix_copilot.kb_service.native_depth_audit import _generated_prose, _judged_evidence, _prose + from infermatrix_copilot.knowledge_service.lifecycle import DEPTH_BLOCK + + pin = "a" * 64 + (tmp_path / "library.py").write_text("def public_value(): return 7\n") + feature = SimpleNamespace(id="public-api", title="Public API") + section = {"facet": "api", "title": "Constant return contract", "interpretation": "fact", + "body": "The public_value entry takes no arguments and returns the constant integer seven.", + "evidence": [{"path": "library.py", "start": 1, "end": 1}]} + block = render_block(feature, section, tmp_path, repository, pin, acceptance_mode="lightweight") + assert DEPTH_BLOCK.fullmatch(block.rstrip("\n")) is not None + assert _generated_prose(section, repository, pin) == _prose(block) + _judged_evidence({"evidence": [{"kind": "upstream_text", "text": ["1: def public_value(): return 7"], + "source_reference": repository + "@" + pin + ":library.py:L1-L1"}]}, block, repository, pin, "api") diff --git a/test/test_portable_publication.py b/test/test_portable_publication.py new file mode 100644 index 000000000..6809a1b2c --- /dev/null +++ b/test/test_portable_publication.py @@ -0,0 +1,89 @@ +"""Activation receipt checks on real snapshot bytes; no model calls are simulated.""" +import hashlib +import json + +import pytest +import yaml + +from infermatrix_copilot.kb_service.activate import verify_snapshot +from infermatrix_copilot.kb_service.knowledge_store import KnowledgeStore, KnowledgeStoreError +from infermatrix_copilot.kb_service.portable_publication import check_publication +from infermatrix_copilot.kb_service.repo_spec import RepoSpec +from infermatrix_copilot.knowledge_view import KnowledgeViewError, build_manifest + + +PIN = "a" * 40 +HEAD = "b" * 40 + + +def _snapshot(tmp_path, tier="final"): + root = tmp_path / "canonical" / "knowledge" + catalog = "schema_version: 1\nfixture_catalog: fixed\n" + catalog_hash = hashlib.sha256(catalog.encode()).hexdigest() + spec = RepoSpec("library", source_pin=PIN, catalog_hash=catalog_hash, policy_hash=catalog_hash) + def page(title): + return (f'---\ntitle: "{title}"\ncreated: 2026-10-06\nupdated: 2026-10-06\n' + f'type: index\ntags: [general]\nsources: []\n---\n\n# {title}\n') + files = {"AGENTS.md": "Knowledge governance\n", "_format.yaml": "format_version: 2\n", + "_repositories.yaml": yaml.safe_dump({"schema_version": 1, "repos": {"library": spec.to_dict()}}), + "general/_index.md": page("General"), "repos/_index.md": page("Repositories"), + "repos/library/_index.md": page("Library"), "repos/library/_catalog.yaml": catalog} + for rel, text in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + summary = {"tier": tier, "init_complete": tier == "final", "source_pin": PIN, + "catalog_sha256": catalog_hash, + "checks": {"source": True, "native": True, "structural": True, "format": True, + "retrieval": True, "semantic_depth": tier == "final"}} + snapshot = tmp_path / "snapshot" + KnowledgeStore(root, "library").snapshot(snapshot, source_pin=PIN, publication_head=HEAD, acceptance=summary) + return snapshot + + +def _rebind_bytes(snapshot): + """Rebuild format integrity only; never renew the publication acceptance scope.""" + manifest = json.loads((snapshot / "MANIFEST.json").read_text()) + manifest.update(build_manifest(snapshot / "knowledge", manifest["snapshot"])) + (snapshot / "MANIFEST.json").write_text(json.dumps(manifest)) + + +def test_final_publication_acceptance_is_bound_to_verified_snapshot(tmp_path): + view = verify_snapshot(_snapshot(tmp_path)) + assert check_publication(view) == {"tiers": {"library": "final"}, "init_complete": True} + + +def test_foundation_activation_requires_explicit_partial_choice(tmp_path): + view = verify_snapshot(_snapshot(tmp_path, "foundation")) + with pytest.raises(KnowledgeStoreError, match="incomplete"): + check_publication(view) + assert check_publication(view, allow_partial=True) == { + "tiers": {"library": "foundation"}, "init_complete": False} + + +def test_rehashing_format_manifest_does_not_renew_publication_scope(tmp_path): + snapshot = _snapshot(tmp_path) + (snapshot / "knowledge/repos/library/_index.md").write_text( + (snapshot / "knowledge/repos/library/_index.md").read_text() + "Changed after acceptance.\n") + _rebind_bytes(snapshot) + view = verify_snapshot(snapshot) + with pytest.raises(KnowledgeStoreError, match="no longer binds"): + check_publication(view) + + +def test_catalog_drift_is_refused_even_with_new_format_hashes(tmp_path): + snapshot = _snapshot(tmp_path) + (snapshot / "knowledge/repos/library/_catalog.yaml").write_text("Changed catalog.\n") + _rebind_bytes(snapshot) + view = verify_snapshot(snapshot) + with pytest.raises(KnowledgeStoreError, match="identity"): + check_publication(view) + + +def test_verified_view_rejects_receipt_byte_substitution(tmp_path): + snapshot = _snapshot(tmp_path) + view = verify_snapshot(snapshot) + receipt = snapshot / "knowledge/_publication-library.json" + receipt.write_text(receipt.read_text() + "\n") + with pytest.raises(KnowledgeViewError): + check_publication(view) diff --git a/test/test_thin_mcp_server.py b/test/test_thin_mcp_server.py index 0f93ebe77..fad4f7f26 100644 --- a/test/test_thin_mcp_server.py +++ b/test/test_thin_mcp_server.py @@ -110,6 +110,11 @@ def test_direct_entrypoints_do_not_resolve_repo(monkeypatch): "update_knowledge", "doc_search", "doc_read", + "open_knowledge_context", + "read_knowledge_context", + "search_knowledge_context", + "expand_knowledge_context", + "related_knowledge_context", } review = mcp.tools["review"]( diff --git a/tools/audit_native_depth_approvals.py b/tools/audit_native_depth_approvals.py index 97e0919f5..7e2a4e5b2 100644 --- a/tools/audit_native_depth_approvals.py +++ b/tools/audit_native_depth_approvals.py @@ -1,431 +1,14 @@ #!/usr/bin/env python3 -"""Verify current depth approvals against archived native model calls and blobs.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import re -from bisect import bisect_right +"""Compatibility CLI for the packaged native depth approval auditor.""" from pathlib import Path -from urllib.parse import quote - -import yaml - -from infermatrix_copilot.kb_service.gate import DIMENSIONS -from infermatrix_copilot.kb_service.init_stages import neutral_headings -from infermatrix_copilot.kb_service.knowledge_coverage import load_policy, policy_path -from infermatrix_copilot.kb_service.knowledge_depth import depth_page, depth_acceptance_mode, validate_draft -from infermatrix_copilot.kb_service.models import ModelUnavailable, parse_json_object -from infermatrix_copilot.knowledge_service.lifecycle import ( - DEPTH_BLOCK, DEPTH_FACETS, depth_proof_basis, depth_sections, safe_source_path, -) -from infermatrix_copilot.trace_store import TraceStore - -MODEL_FIELDS = ("role", "provider", "model", "effort", "served_model") - - -def _sha(value: bytes) -> str: - return hashlib.sha256(value).hexdigest() - - -def _payload(text: str) -> dict: - match = re.fullmatch(r"\s*(.*?)\s*\s*", text, re.S) - data = json.loads(match[1] if match else text) - if not isinstance(data, dict): - raise ValueError("native prompt must contain an object") - return data - - -def _prose(block: str) -> str: - # This is exactly the rendering used by depth_judge.review_facets. - return re.sub(r"", "", block, flags=re.S).strip() - - -def _family(model: str) -> str: - return model.casefold().split("-")[0] - - -def _generated_prose(section: dict, repository: str, pin: str) -> str: - """Replay only render_block's deterministic lightweight prose transforms.""" - validate_draft({"sections": [section]}, acceptance_mode="lightweight") - title = " ".join(str(section.get("title") or section["facet"]).replace("#", "").split()) - body = neutral_headings(f"## {title}\n\n") - if section.get("validation_kind") == "documented_manual": - body += "文档中的人工验收步骤（本轮未执行）：\n\n" - if section["interpretation"] == "inference": - body += "设计推断（非作者历史意图）：\n\n" - body += section["body"].strip() + "\n\n" - body += "来源：" + ", ".join( - f"[{e['path']}:L{e['start']}–L{e['end']}](https://github.com/{repository}/blob/{pin}/" - f"{quote(e['path'], safe='/')}#L{e['start']}-L{e['end']})" for e in section["evidence"]) + "\n" - return _prose(body) - - -def _generator_evidence(data: dict, repository: str, pin: str) -> dict: - """Convert the actual offered files/docs to the existing exact-span verifier.""" - packet = [] - for kind in ("files", "docs"): - items = data.get(kind, []) - if not isinstance(items, list): - raise ValueError("native generator offered spans are malformed") - for item in items: - if not isinstance(item, dict) or not safe_source_path(item.get("path")) \ - or type(item.get("start")) is not int or type(item.get("end")) is not int \ - or not 1 <= item["start"] <= item["end"] \ - or not isinstance(item.get("text"), list) \ - or len(item["text"]) != item["end"] - item["start"] + 1: - raise ValueError("native generator offered spans are malformed") - rows = item["text"] if kind == "files" else [f"{n}: {line}" for n, line in enumerate(item["text"], item["start"])] - if any(not isinstance(line, str) for line in item["text"]): - raise ValueError("native generator offered lines are malformed") - packet.append({"kind": "upstream_text", "text": rows, - "source_reference": f"{repository}@{pin}:{item['path']}:L{item['start']}-L{item['end']}"}) - return {"evidence": packet} - - -def _lightweight_generator(candidates, records, *, before: float, feature: str, facet: str, - block: str, proof: dict, repository: str, pin: str) -> dict: - """Find the actual retained draft, which may precede a later unrelated retry.""" - expected = [{key: e[key] for key in ("path", "start", "end")} for e in proof["evidence"]] - for candidate in reversed(candidates): - native = records.get(candidate["id"]) - if candidate["at"] > before or native is None: - continue - record, store = native - model = record.get("model", {}) - requested, served = model.get("model"), model.get("served_model") - if model.get("provider") != "zcode" or not isinstance(requested, str) or requested.casefold() != "glm-5.3" \ - or model.get("fallback_from") or record.get("result", {}).get("fallback_from") \ - or served not in (None, "") and (not isinstance(served, str) or served.casefold() != "glm-5.3"): - continue - try: - data = _payload(store.blob(record["inputs"]["prompt"])) - requested_facets = data.get("facets") - if data.get("repository") != repository or data.get("feature", {}).get("id") != feature \ - or data.get("pin") != pin or data.get("acceptance_mode") != "lightweight" \ - or not isinstance(requested_facets, list) or requested_facets.count(facet) != 1: - continue - draft = parse_json_object(store.blob(record["outputs"]["reply"])) - sections = draft.get("sections") - if not isinstance(sections, list): - continue - matching = [section for section in sections if isinstance(section, dict) and section.get("facet") == facet] - if len(matching) != 1: - continue - section = matching[0] - spans = [{key: e[key] for key in ("path", "start", "end")} for e in section["evidence"]] - if spans != expected or section.get("validation_kind") != proof.get("validation_kind") \ - or _generated_prose(section, repository, pin) != _prose(block): - continue - _judged_evidence(_generator_evidence(data, repository, pin), block, repository, pin, facet) - return record - except (OSError, ValueError, TypeError, KeyError, AttributeError, ModelUnavailable): - continue - raise ValueError("no exact successful Zcode GLM-5.3 lightweight draft and offered evidence before this judgment") - - -def _judged_evidence(data: dict, block: str, repository: str, pin: str, facet: str) -> None: - """Bind the actual judge context to each persisted source witness. - - Historical packets merge adjoining evidence spans and omit line endings. - Source-span hashes normalize CRLF through read_text, so replay their UTF-8 - text with either a final LF or no terminator for a source's final line. - """ - match = re.search(r"\n\s*\n$", block, re.S) - if match is None: - raise ValueError("current facet has no evidence proof") - proof = json.loads(match[1]) - basis = depth_proof_basis(proof, facet=facet, pin=pin) - evidence = data.get("evidence") - if not isinstance(evidence, list) or not evidence: - raise ValueError("native judgment has no source evidence packet") - lines, certificates = {}, [] - for item in evidence: - if not isinstance(item, dict): - raise ValueError("native judgment has malformed evidence") - if item.get("kind") == "replayed_absence_certificate": - certificates.append(item.get("certificate")) - continue - if item.get("kind") != "upstream_text": - raise ValueError("native judgment has an unknown evidence kind") - reference = item.get("source_reference") - prefix = repository + "@" + pin + ":" - if not isinstance(reference, str) or not reference.startswith(prefix): - raise ValueError("native evidence reference differs from the repository or pin") - span = re.fullmatch(r"(.+):L([1-9][0-9]*)-L([1-9][0-9]*)", reference[len(prefix):]) - if span is None or not safe_source_path(span[1]): - raise ValueError("native evidence has an invalid source reference") - path, start, end = span[1], int(span[2]), int(span[3]) - text = item.get("text") - if end < start or not isinstance(text, list) or len(text) != end - start + 1: - raise ValueError("native numbered evidence does not cover its declared span") - for number, row in enumerate(text, start): - marker = f"{number}: " - if not isinstance(row, str) or not row.startswith(marker) or "\n" in row or "\r" in row: - raise ValueError("native numbered evidence is malformed or noncontiguous") - value = row[len(marker):] - key = (path, number) - if key in lines and lines[key] != value: - raise ValueError("native evidence contains conflicting source lines") - lines[key] = value - for span in proof["evidence"]: - try: - shown = "\n".join(lines[(span["path"], number)] for number in range(span["start"], span["end"] + 1)) - except KeyError as exc: - raise ValueError("native judgment omitted a current proof span") from exc - if span["sha256"] not in {_sha(shown.encode()), _sha((shown + "\n").encode())}: - raise ValueError("native source text differs from its current proof hash") - if basis == "verified_absent" and certificates != [proof["absence_certificate"]]: - raise ValueError("native judgment omitted or changed the replayed absence certificate") - - -def _archives(paths: list[Path]) -> tuple[dict, dict, list[dict], list[str]]: - records, generators, provenance, problems = {}, {}, [], [] - for root in paths: - store = TraceStore(root) - record_files = sorted((root / "records").glob("*.jsonl")) - if not record_files: - problems.append(f"{root.name}: no native trace record archive") - continue - record_bytes = {path: path.read_bytes() for path in record_files} - hashed = hashlib.sha256() - blob_files = sorted((root / "blobs").rglob("*.gz")) - for path in sorted(record_files + blob_files): - raw = record_bytes[path] if path in record_bytes else path.read_bytes() - hashed.update(path.relative_to(root).as_posix().encode() + b"\0" + raw) - provenance.append({"name": root.name, "archive_sha256": hashed.hexdigest(), - "record_files": len(record_files), "blob_files": len(blob_files)}) - for path in record_files: - for number, line in enumerate(record_bytes[path].decode().splitlines(), 1): - try: - record = json.loads(line) - if not isinstance(record, dict) or record.get("schema") != "trace/1" \ - or not isinstance(record.get("id"), str) or not record["id"]: - raise ValueError("invalid native trace record") - record_id = record["id"] - if record_id in records: - problems.append(f"duplicate native trace ID: {record_id}") - records[record_id] = None - continue - records[record_id] = (record, store) - if record.get("kind") != "model_call" or record.get("error") != "" \ - or record.get("model", {}).get("role") != "generator": - continue - data = _payload(store.blob(record["inputs"]["prompt"])) - feature = data.get("feature") - feature = feature.get("id") if isinstance(feature, dict) else feature - pin = data.get("pin") - if isinstance(feature, str) and isinstance(pin, str) \ - and isinstance(record.get("at"), (int, float)): - generators.setdefault((feature, pin), []).append(record) - except (OSError, ValueError, TypeError, KeyError, AttributeError) as exc: - problems.append(f"{root.name}/{path.name}:{number}: unreadable native record or blob ({type(exc).__name__})") - for rows in generators.values(): - rows.sort(key=lambda row: (row["at"], row["id"])) - return records, generators, provenance, problems - - -def _receipts(baselines: list[Path], checkpoints: list[Path], pages: dict) -> tuple[dict, list[dict], list[str]]: - receipts, provenance, problems = {}, [], [] - for kind, paths in (("baseline_report", baselines), ("checkpoint", checkpoints)): - for path in paths: - try: - raw = path.read_bytes() - data = json.loads(raw) - provenance.append({"kind": kind, "name": path.name, "sha256": _sha(raw)}) - if kind == "baseline_report": - rows = data.get("approvals", data.get("native_execution", {}).get("approvals")) - if not isinstance(rows, list): - raise ValueError("baseline report must contain an approvals list") - else: - rows = [] - for key, verdict in data.get("verdicts", {}).items(): - if not key.startswith("depth:"): - continue - feature = key.removeprefix("depth:") - for facet, result in verdict.get("facets", {}).items(): - if result.get("verdict") == "pass": - rows.append({**result, "feature": feature, "facet": facet, - "page": pages.get(feature), "pin": data.get("pin")}) - for row in rows: - if not isinstance(row, dict) or not isinstance(row.get("feature"), str) \ - or row.get("facet") not in DEPTH_FACETS: - raise ValueError("malformed native approval receipt") - key = (row["feature"], row["facet"]) - if key in receipts: - problems.append("duplicate native approval receipt: " + "/".join(key)) - receipts[key] = None - else: - receipts[key] = row - except (OSError, ValueError, TypeError, AttributeError) as exc: - problems.append(f"{path.name}: unreadable {kind} ({type(exc).__name__})") - return receipts, provenance, problems - - -def audit_native_approvals(root: Path, repo: str, *, baseline_reports: list[Path], - trace_dirs: list[Path], checkpoints: list[Path]) -> dict: - """No model calls or archive/index writes; reports contain no raw payloads. - - This verifies native judgment bindings. The separate pinned depth auditor - owns source-span, flow and absence-certificate replay against upstream. - """ - root = root.resolve() - if not re.fullmatch(r"[a-z0-9][a-z0-9._-]*", repo): - raise ValueError("repo must be a canonical adapter slug") - manifest = yaml.safe_load((root / f"adapters/{repo}/manifest.yaml").read_text()) - repository = manifest.get("repo", {}).get("full_name") - if not isinstance(repository, str) or not re.fullmatch(r"[^/@:]+/[^/@:]+", repository): - raise ValueError("adapter must declare its source repository full_name") - policy_file = root / policy_path(repo) - policy = load_policy(policy_file.read_text(), manifest["knowledge"]["repo_subdir"]) - pages = {feature.id: depth_page(feature) for feature in policy.features} - records, generators, archive_provenance, problems = _archives(trace_dirs) - receipts, receipt_provenance, receipt_problems = _receipts(baseline_reports, checkpoints, pages) - problems += receipt_problems - approvals, current_keys, knowledge_hash = [], set(), hashlib.sha256() - dimensions = {key: "yes" for key in DIMENSIONS["prose"]} - for feature in policy.features: - page = pages[feature.id] - path = root / "knowledge" / page - if not path.is_file(): - continue - text = path.read_text() - knowledge_hash.update(page.encode() + b"\0" + text.encode()) - matches = list(DEPTH_BLOCK.finditer(text)) - valid = {(row["feature"], row["facet"]) for row in depth_sections(text)} - if text.count("", match[0], re.S)[1]) - mode = depth_acceptance_mode(proof) - receipt = receipts.get(key) - if not receipt or receipt.get("page") != page or receipt.get("block_sha256") != block_hash \ - or receipt.get("pin", pin) != pin: - raise ValueError("missing or mismatched current-block approval receipt") - if receipt.get("acceptance_mode", "strict") != mode: - raise ValueError("approval receipt acceptance mode differs from current block") - native = records.get(receipt.get("native_trace_id")) - if not native: - raise ValueError("missing or duplicate native judgment trace") - record, store = native - model = record.get("model", {}) - if record.get("kind") != "model_call" or record.get("error") != "" \ - or model.get("role") != "judge" or model.get("provider") != "codex" \ - or not isinstance(model.get("model"), str) or not model["model"]: - raise ValueError("native record is not a successful Codex judge call") - served = model.get("served_model") - if served not in (None, "") and (not isinstance(served, str) - or not re.match(r"^(?:gpt-|codex(?:-|$)|o[0-9]+(?:-|$))", served, re.I)): - raise ValueError("native record reports an incompatible Codex judge served identity") - reply_ref = record.get("outputs", {}).get("reply") - expected_ref = "sha256:" + str(receipt.get("native_reply_sha256", "")) - if reply_ref != expected_ref: - raise ValueError("native reply hash differs from approval receipt") - response = parse_json_object(store.blob(reply_ref)) - result = response.get("facets", {}).get(facet) - if not isinstance(result, dict) or result.get("dimensions") != dimensions \ - or not isinstance(result.get("reason"), str): - raise ValueError("native facet judgment is not exactly three yes dimensions") - data = _payload(store.blob(record["inputs"]["prompt"])) - if data.get("feature") != fid or data.get("pin") != pin \ - or data.get("sections", {}).get(facet) != _prose(match[0]): - raise ValueError("native judgment prompt does not bind the exact current facet prose and pin") - modes = data.get("acceptance_modes", {}) - if not isinstance(modes, dict) or modes.get(facet, "strict") != mode: - raise ValueError("native judgment prompt does not bind the current acceptance mode") - validation_kind = proof.get("validation_kind") - if validation_kind is not None and (receipt.get("validation_kind") != validation_kind or - data.get("validation_kinds", {}).get(facet) != validation_kind): - raise ValueError("native judgment and receipt do not bind the validation category") - _judged_evidence(data, match[0], repository, pin, facet) - sections, results = data["sections"], response["facets"] - if mode == "strict" and (set(sections) != set(results) or any( - name not in DEPTH_FACETS or not isinstance(item, dict) - or not isinstance(item.get("reason"), str) or not isinstance(item.get("dimensions"), dict) - or set(item["dimensions"]) != set(dimensions) - or any(value not in ("yes", "no", "unsure") for value in item["dimensions"].values()) - for name, item in results.items())): - raise ValueError("native response does not answer the complete judged packet") - candidates = generators.get((fid, pin), []) - index = bisect_right([row["at"] for row in candidates], record["at"]) - 1 - if index < 0: - raise ValueError("no successful native generator context before this judgment") - generator = _lightweight_generator(candidates, records, before=record["at"], feature=fid, - facet=facet, block=match[0], proof=proof, repository=repository, pin=pin) \ - if mode == "lightweight" else candidates[index] - if records.get(generator["id"]) is None: - raise ValueError("generator trace ID is duplicated") - generator_model = generator.get("model", {}) - gen_name = generator_model.get("served_model") or generator_model.get("model") - judge_name = model.get("served_model") or model["model"] - if not isinstance(gen_name, str) or not gen_name or not isinstance(judge_name, str) \ - or not isinstance(generator_model.get("provider"), str) or not generator_model["provider"] \ - or _family(gen_name) == _family(judge_name): - raise ValueError("native generator and judge are not independently identified model families") - approvals.append({"feature": fid, "facet": facet, "page": page, "pin": pin, - "acceptance_mode": mode, - **({"validation_kind": validation_kind} if validation_kind is not None else {}), - "block_sha256": block_hash, "dimensions": result["dimensions"], - "native_trace_id": record["id"], "native_reply_sha256": reply_ref.removeprefix("sha256:"), - "model": {key: model.get(key, "") for key in MODEL_FIELDS}, - "native_generator_trace_id": generator["id"], - "generator_model": {key: generator_model.get(key, "") for key in MODEL_FIELDS}}) - except (OSError, ValueError, TypeError, KeyError, AttributeError, ModelUnavailable) as exc: - detail = str(exc) if isinstance(exc, ValueError) else type(exc).__name__ - problems.append(label + ": " + detail) - for key in sorted(set(receipts) - current_keys): - problems.append("approval receipt has no current depth block: " + "/".join(key)) - approvals.sort(key=lambda row: (row["feature"], DEPTH_FACETS.index(row["facet"]))) - return {"schema_version": 1, "repo": repo, "approvals": approvals, "problems": problems, - "provenance": {"policy_sha256": _sha(policy_file.read_bytes()), - "depth_pages_sha256": knowledge_hash.hexdigest(), "trace_archives": archive_provenance, - "input_reports": receipt_provenance, - "approval_set_sha256": _sha(json.dumps(approvals, ensure_ascii=False, sort_keys=True).encode())}, - "interpretation": "Three yes dimensions were read from native reply blobs bound to current prose. " - "Unreported served-model identities remain unreported; requested identity and provider " - "come from the native record. Lightweight generator provenance also binds an exact " - "Zcode GLM-5.3 draft and its actually offered source/document spans without fallback; " - "strict historical provenance retains its existing model-family check. " - "Source and absence proofs require the pinned depth audit."} - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - parser.add_argument("--repo", required=True) - parser.add_argument("--baseline-report", type=Path, action="append", default=[]) - parser.add_argument("--trace-dir", type=Path, action="append", required=True) - parser.add_argument("--checkpoint", type=Path, action="append", default=[]) - parser.add_argument("--report", type=Path) - args = parser.parse_args(argv) - if args.report and args.report.resolve().is_relative_to((args.root / "knowledge").resolve()): - parser.error("approval reports belong outside product knowledge") - try: - report = audit_native_approvals(args.root, args.repo, baseline_reports=args.baseline_report, - trace_dirs=args.trace_dir, checkpoints=args.checkpoint) - except (OSError, ValueError, TypeError, KeyError) as exc: - parser.error(f"cannot read native approval inputs: {type(exc).__name__}") - raw = json.dumps(report, ensure_ascii=False, indent=2) + "\n" - if args.report: - args.report.parent.mkdir(parents=True, exist_ok=True) - args.report.write_text(raw) - print(json.dumps({"approvals": len(report["approvals"]), "problems": report["problems"], - "report_sha256": _sha(raw.encode())}, ensure_ascii=False, indent=2)) - return 1 if report["problems"] else 0 +import sys +from infermatrix_copilot.kb_service import native_depth_audit as _implementation +globals().update({key: value for key, value in vars(_implementation).items() + if not key.startswith("__")}) if __name__ == "__main__": - raise SystemExit(main()) + arguments = sys.argv[1:] + if "--root" not in arguments: + arguments = ["--root", str(Path(__file__).resolve().parents[1]), *arguments] + raise SystemExit(_implementation.main(arguments)) From 57ca043105112d1e887696402b7c6d11252d3f13 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Tue, 6 Oct 2026 15:00:34 +0800 Subject: [PATCH 2/2] Document SHA256 pinned source compatibility --- doc/architecture/SPEC/knowledge_service.md | 6 +++++- doc/architecture/SPEC/knowledge_service/lifecycle.md | 4 +++- doc/architecture/SPEC/knowledge_service/pinned_claims.md | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/doc/architecture/SPEC/knowledge_service.md b/doc/architecture/SPEC/knowledge_service.md index 6b0dd8d15..c4f6000b1 100644 --- a/doc/architecture/SPEC/knowledge_service.md +++ b/doc/architecture/SPEC/knowledge_service.md @@ -1,6 +1,10 @@ # knowledge_service/ — provider curation components - + + +Pinned source identities and depth markers accept full SHA-1 or SHA-256 Git +commits. Evidence bodies, stored hashes and native approval semantics remain +unchanged; accepting a wider object identity never renews an old approval. The provider owns knowledge curation beneath the public SDK v1 facade. `KnowledgeCurator` composes four domain components over one explicit work diff --git a/doc/architecture/SPEC/knowledge_service/lifecycle.md b/doc/architecture/SPEC/knowledge_service/lifecycle.md index 366f5b355..533e7a763 100644 --- a/doc/architecture/SPEC/knowledge_service/lifecycle.md +++ b/doc/architecture/SPEC/knowledge_service/lifecycle.md @@ -1,6 +1,8 @@ # knowledge_service/lifecycle.py —— 规范 - + + +深度区块的固定源码身份支持完整 SHA-1 或 SHA-256；旧区块正文、哈希及证明语义保持不变。 深度验证入口缺失证书的当前检测器为 `static-test-association-v2`；解析、证书工厂和 检索形状校验共用 `DEPTH_ABSENCE_DETECTOR`，算法升级不复用旧检测器身份。 diff --git a/doc/architecture/SPEC/knowledge_service/pinned_claims.md b/doc/architecture/SPEC/knowledge_service/pinned_claims.md index 7b6c4ac94..757d5539e 100644 --- a/doc/architecture/SPEC/knowledge_service/pinned_claims.md +++ b/doc/architecture/SPEC/knowledge_service/pinned_claims.md @@ -1,6 +1,6 @@ # knowledge_service/pinned_claims.py —— 规范 - + `LOC ~200 · 在钉住的上游 SHA 上核验规则声明与代码证据（kb init §9.1） · refactor-status: new` @@ -8,7 +8,7 @@ - `PinnedObserver(repo_dir, repository, pin, pull=)`：在本地 git 仓库（bare 或非 bare）上实现 `facts.Observer`，`head()` 固定返回钉住的 SHA；`top_level` / `path_exists` / `file_text` 读 git 对象，`pull(n)` 默认走 `gh api repos/{repo}/pulls/{n}`（可注入；任何查询失败——没有 `gh`、网络错误、非对象回答——都归一为 `FactsError`，原有的 `FactsError` 原样抛出）。另加协议之外的 - `is_ancestor(sha)`：`sha` 是钉点或其祖先（非 40 位 SHA、镜像中不存在的对象、非 commit 对象为 False；候选对象损坏（`cat-file --batch-check` 同样输出 missing 但 stderr 报解包错误），或 git 读不到钉点历史（`merge-base` 此时也以 1 退出、只在 stderr 报 `Could not read`）时抛 `FactsError`，不把读失败当成"合入在钉点之后"）。钉点必须是 40 位 + `is_ancestor(sha)`：`sha` 是钉点或其祖先（非完整 40/64 位 SHA、镜像中不存在的对象、非 commit 对象为 False；候选对象损坏（`cat-file --batch-check` 同样输出 missing 但 stderr 报解包错误），或 git 读不到钉点历史（`merge-base` 此时也以 1 退出、只在 stderr 报 `Could not read`）时抛 `FactsError`，不把读失败当成"合入在钉点之后"）。钉点必须是完整 40/64 位 且在仓库中存在的提交，否则构造即抛 `FactsError`。 - `check_rules(rule_texts, observer)`：对每条（活跃）规则用 `facts.claims_in` 抽取声明，按 `facts.observe` 在钉点观测；每条声明必须成立，`^[PR #N]` 还必须已合并进钉点历史