From cff5aceaf285d0126399f466ca844d5f2c51aad5 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:30:54 +0800 Subject: [PATCH 01/14] docs(rfc): specify bounded memory entry pagination --- .../1656_bounded_memory_entry_pagination.md | 241 ++++++++++++++++++ .../1656_bounded_memory_entry_pagination.md | 213 ++++++++++++++++ 2 files changed, 454 insertions(+) create mode 100644 docs/en/rfcs/1656_bounded_memory_entry_pagination.md create mode 100644 docs/zh/rfcs/1656_bounded_memory_entry_pagination.md diff --git a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md new file mode 100644 index 0000000000..91480608e1 --- /dev/null +++ b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md @@ -0,0 +1,241 @@ +- Proposal Name: `bounded_memory_entry_pagination` +- Start Date: 2026-09-28 +- Tracking Issue: [oceanbase/powercontext#1656](https://github.com/oceanbase/powercontext/issues/1656) +- Related work: [#1657](https://github.com/oceanbase/powercontext/issues/1657), + [#1709](https://github.com/oceanbase/powercontext/pull/1709), and + [#1718](https://github.com/oceanbase/powercontext/issues/1718) +- Status: core design accepted in principle; clarified contract specified; implementation not started + +# Summary + +PowerContext adds an additive, bounded Memory-entry directory query. The existing full-result +`list_memory_entries` operation remains unchanged. The new operation returns exact entry identities and compact +metadata in `entry_id ASC` order, without loading entry bodies, and continues with a signed opaque cursor. + +Page one pins an immutable Memory revision. Later pages query a rebuildable revision-valid directory index at that +revision, so concurrent Memory writes cannot cause duplicates or omissions. Tag-filtered pages also bind a separate +tag generation because tag assignments can change without advancing the Memory revision. A tag change explicitly +invalidates continuation and requires a restart. + +# Part A: SPEC + +## 1. Background and problem + +The current Memory list path resolves the latest Memory Artifact, decodes its complete manifest, loads every referenced +entry-version row, and only then applies presentation and tag filtering. Its persistence and response work therefore +grow with the complete Memory even when a caller only needs a small inventory page. HTTP-level slicing would bound the +returned JSON but not database work, manifest decoding, or entry-body expansion. + +Remote clients, MCP integrations, dashboards, and audit tools need a stable, bounded inventory before choosing entries +to retrieve through the existing exact-detail operation. + +## 2. Target behavior + +Add `POST /v1/memory/entries/query` with operation ID `query_memory_entries`. + +The request contains `scope_id`, `include_inactive` (default `false`), the existing optional `tag_filter`, `limit` +(default `50`, range `1..100`), and an optional opaque `cursor`. + +The response contains the pinned `memory_ref` (or `null` when the Scope has no Memory), `items` ordered by +`entry_id ASC`, and `next_cursor`, where `null` means traversal is complete. Each item contains only the exact citation +(`memory_ref`, `entry_id`, and `entry_version_id`), positive logical `version`, `kind`, and lifecycle `state`. It does +not contain `text`, `source_refs`, or `artifact_refs`; callers use exact detail for the body. + +The request, response, ordering, and behavior of `list_memory_entries` do not change. + +## 3. Main scenarios + +### Start and continue an unfiltered traversal + +1. The Server authenticates the caller and authorizes the Scope. +2. With no cursor, it resolves and pins the current Memory revision. If no Memory exists, it returns a null reference, + empty items, and no cursor. +3. It reads eligible compact directory rows after the page key and returns at most the count and byte budgets. +4. When more rows exist, the signed cursor continues after the last emitted `entry_id`. +5. Every continuation is authorized again and reads the same pinned revision. +6. Concurrent Memory commits remain invisible to this traversal; a new traversal sees the new head. + +### Apply lifecycle and tag filters + +Revision validity, lifecycle state, and tag predicates are applied before keyset paging and the item limit. Page one of +a tag-filtered query captures the current durable tag generation. A continuation validates it before and after its +bounded read. An effective Memory-entry tag mutation causes `400 invalid_cursor` with reason `tag_state_changed`; the +caller restarts at page one. A no-op tag replacement does not advance generation, and unfiltered cursors do not depend +on tag generation. This is invalidation/restart, not a tag snapshot guarantee. + +### Existing database without a verified index + +After authentication and authorization, the new operation returns `503 memory_query_index_unavailable`. It never +falls back to the legacy full-manifest path. Legacy list, exact detail, search, and Memory writes remain available +outside a declared maintenance window. + +The operator runs an explicit offline `plan`, bounded/resumable `apply`, and `verify` workflow for SQLite or OceanBase. +Only successful verification marks the feature ready. Interrupted apply remains resumable and never rewrites +authoritative Memory data. Outside the maintenance window, new Memory commits maintain their directory deltas even +while historical backfill is incomplete, but the query remains unavailable until full verification succeeds. + +### Enforce a page-byte limit + +The Server measures encoded directory-item payloads against a fixed 4 MiB budget. It stops before an additional item +would exceed either limit and continues after the last emitted key. Legal v1 fields are already bounded. If corrupt +legacy data or a future incompatible shape creates one item that cannot fit, the operation returns +`413 memory_directory_item_too_large` with no partial page and no advanced cursor. + +## 4. Rules and invariants + +| ID | Rule or invariant | Result when violated | +|---|---|---| +| MEM-PAGE-01 | The new operation is additive; legacy full listing is unchanged. | Compatibility regression. | +| MEM-PAGE-02 | A traversal is pinned to one immutable Memory revision. | Invalid cursor or implementation defect. | +| MEM-PAGE-03 | Eligible rows use `entry_id ASC`; filtering precedes paging. | Skip, duplicate, or order regression. | +| MEM-PAGE-04 | Every item preserves exact Memory, entry, and entry-version identity. | Citation integrity failure. | +| MEM-PAGE-05 | Directory reads never decode bodies or expand body references. | Bounded-work contract failure. | +| MEM-PAGE-06 | A cursor binds endpoint, Scope, filters, limit, order version, pinned revision, and applicable tag generation. | `400 invalid_cursor`. | +| MEM-PAGE-07 | Authentication and Scope authorization run on every page before private existence or readiness is disclosed. | Access-control defect. | +| MEM-PAGE-08 | Effective Memory-entry tag mutations advance one generation atomically; no-op replacements do not. | Tag transaction rolls back. | +| MEM-PAGE-09 | Tag generation changes invalidate filtered, but not unfiltered, continuation. | `400 invalid_cursor`, `tag_state_changed`. | +| MEM-PAGE-10 | The query index is rebuildable derived state; Artifacts, manifests, entry versions, and tag rows remain authoritative. | Verification fails. | +| MEM-PAGE-11 | Directory deltas commit atomically with Memory and are proportional to changed entries. | Whole transaction rolls back. | +| MEM-PAGE-12 | An unverified index never serves or triggers an unbounded fallback. | `503 memory_query_index_unavailable`. | +| MEM-PAGE-13 | A page contains at most 100 items and 4 MiB of encoded item payload. | Page ends before the next item. | +| MEM-PAGE-14 | One over-budget item cannot produce partial success. | `413 memory_directory_item_too_large`. | + +## 5. Failure, retry, and recovery semantics + +- Malformed, tampered, cross-endpoint, cross-Scope, or request-mismatched cursors use the existing + `400 invalid_cursor` contract. Expired signed cursors use `410 cursor_expired`. +- Retrying the same `tag_state_changed` cursor is not useful; restarting page one is safe. +- Existing authentication and authorization errors take precedence over readiness and private Memory existence. +- Retrying `memory_query_index_unavailable` is safe after migration verification. +- An oversized item returns no items and no replacement cursor. +- Memory and tag writes are all-or-nothing: authoritative state and related index/generation deltas commit or roll back + together. +- Migration apply commits bounded checkpoints and is resumable; verify alone may mark readiness complete. + +## 6. Concurrency and resource constraints + +- Memory revision pinning provides a stable entry view. Tag rows are not revisioned, so tag-filtered traversal uses + invalidation instead of claiming a snapshot. +- Tag generation is checked around the bounded tag query, closing changes during page assembly. +- The last emitted `entry_id`, not the lookahead row, is the continuation boundary. +- Persistence materializes at most `limit + 1` eligible compact rows and does not deserialize entry-body JSON. +- Item encoding is measured incrementally and capped at 4 MiB, excluding a separately bounded envelope and cursor. +- Query plans and measured work are recorded for 200, 1,000, and 5,000 entries. Evidence distinguishes returned or + materialized rows from database rows examined; it must not claim a bound unsupported by SQLite or OceanBase plans. +- A Memory commit closes and inserts validity rows only for changed manifest pointers or states. Existing search + projection rows for unchanged entries remain untouched, preserving #1709. +- Backfill uses a positive batch size and durable checkpoint; startup never runs an unbounded rebuild. + +## 7. Non-goals + +This PR does not paginate or alter `list_memory_entries`; return entry bodies in the directory operation; implement +#1657 history pagination; implement capacity, retention, split/routing, compaction, or physical erasure policy from +#1321/#1718; snapshot all tag assignments; change search, exact-detail, or citation semantics; make derived rows +authoritative; or support mixed old/new writers during the offline migration window. + +## 8. Acceptance criteria + +| Obligation | Observable evidence | Verification layer | +|---|---|---| +| Additive contract | New OpenAPI operation exists; legacy schemas remain compatible. | Contract/generation tests | +| Stable traversal | Concurrent writes neither skip nor duplicate pinned rows. | Runtime and HTTP tests | +| Compact exact identity | Every item resolves through exact detail; body fields are absent. | Runtime/mapping tests | +| Filter before page | Lifecycle and tag filters fill from eligible rows. | Persistence tests | +| Tag restart boundary | Effective changes invalidate; no-op changes and unfiltered cursors do not. | Tag/concurrency tests | +| Authorization per page | Revoked access stops continuation without disclosure. | HTTP access tests | +| Bounded read/response | `limit + 1`, no body decode, count/byte bounds, and atomic 413 are observed. | Instrumented persistence/HTTP tests | +| Delta writes | Only changed entries mutate directory/search projections. | Persistence tests | +| Safe migration | Fresh, incomplete, interrupted, resumed, verified, and corrupt states obey the gate. | SQLite/OceanBase migration tests | +| Scale behavior | 200/1,000/5,000 evidence records plans, rows, bytes, write deltas, and batch work. | Reproducible report | + +# Part B: Design report + +## 1. Final data flow + +```text +OpenAPI query_memory_entries + -> Server authentication and Scope authorization + -> ScopedMemoryApplication.query_directory + -> relational directory authority + -> readiness and pinned-revision validation + -> optional tag-generation validation + -> revision-valid filter + keyset + lookahead query + -> optional tag-generation revalidation + -> item-byte budget and signed continuation + -> Server response mapping +``` + +Memory commits use the existing transaction. After comparing previous and current manifest maps, the transaction +updates validity rows only for changed entries. Tag replacement remains owned by `RelationalTagService`; an effective +Memory-entry tag delta advances its dedicated generation under the existing owner-Artifact serialization. + +## 2. Authority and seams + +| Rule or state | Authority | Interface | Evidence | +|---|---|---|---| +| Wire shape | `openapi/powercontext.yaml` | `query_memory_entries` | schema/generation tests | +| Authorization order | Server application | existing Scope authorization | access tests | +| Directory semantics and budget | Memory runtime/persistence | `query_directory` | persistence/runtime tests | +| Cursor signature and TTL | `SignedCursorCodec` | exact-context encode/decode | cursor tests | +| Revision-valid state | relational Memory persistence | directory validity table | database tests | +| Exact bodies and versions | Memory Artifact and entry-version tables | existing exact read | citation tests | +| Mutable tags | `RelationalTagService` and tag tables | predicate + generation | tag tests | +| Readiness/checkpoint | query-index migration module | plan/apply/verify | migration tests | +| Error presentation | Server error mapping | stable status/code/details | HTTP tests | + +The Server does not reimplement pagination, the runtime does not infer authorization, and the cursor codec authenticates +context without owning query semantics. + +## 3. Persistence mechanism + +A derived directory table stores `entry_id`, exact `entry_version_id`, lifecycle `state`, inclusive +`valid_from_revision`, and exclusive nullable `valid_to_revision` per Scope and Memory Artifact. Indexes support +`entry_id` keyset traversal constrained by `valid_from_revision <= pinned_revision` and +`valid_to_revision IS NULL OR valid_to_revision > pinned_revision`. Compact `version` and `kind` come from an indexed +join to the authoritative entry-version row; body JSON is not selected or decoded. + +At revision `R`, a changed entry closes its previous row with `valid_to_revision = R` and inserts its new pointer/state +with `valid_from_revision = R`. New entries only insert; unchanged entries do nothing. This preserves historical +membership without copying a complete manifest per revision. + +The signed cursor contains exactly the schema/order version, endpoint, Scope, normalized filters, limit, pinned Memory +Artifact ID/revision, optional tag generation, exclusive `after_entry_id`, and expiry. + +A small durable tag-generation table stores one monotonic generation per Scope/Memory Artifact. It advances only when +`RelationalTagService.replace` produces an effective normalized delta for a `memory_entry` target. + +A feature-scoped marker stores query-index schema version, readiness, and resumable checkpoint. Fresh databases start +complete. Existing databases remain incomplete until offline verification confirms derived coverage and identities. + +## 4. Accepted trade-offs + +- Bodies require exact-detail round trips; this preserves a truly bounded directory. +- Tag changes restart filtered traversal instead of retaining an unbounded snapshot. +- Existing databases require explicit offline migration for only this new feature. +- Revision-valid rows use more storage than current heads but avoid per-revision manifest copies. +- The initial order is fixed; future orderings need separately versioned index and cursor contracts. + +## 5. Obligation-to-test map + +| Obligation | Owning layer | Test | +|---|---|---| +| Public compatibility | OpenAPI | schema/generation and SDK tests | +| Revision-valid selection | directory persistence | SQLite/OceanBase keyset tests | +| Atomic delta maintenance | Memory commit | rollback and row-delta tests | +| Tag generation | tag persistence | effective/no-op/concurrent mutation tests | +| Cursor binding | cursor/query seam | mismatch, tamper, and expiry tests | +| Auth before disclosure | Server | HTTP/MCP access tests | +| Byte behavior | query authority/Server | boundary and 413 tests | +| Readiness | migration module | fresh/incomplete/resume/verify/corrupt tests | +| Legacy unchanged | legacy list path | existing suite plus compatibility regression | +| Resource claims | persistence | reproducible scale report | + +## 6. Verification plan and current status + +Before review, run focused OpenAPI, cursor, Memory persistence, tag, migration, Server, access-control, and client tests; +SQLite scale measurements; OceanBase evidence when its service is available; generated-code, contract, documentation, +format, lint, and type checks; then `make check` and `make unit-test`. Report unavailable services and skipped checks +separately from passes. + +Current status: this SPEC is checked against the issue discussion and current call paths. No feature code or feature +tests exist yet, and no implementation check is claimed as passing. diff --git a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md new file mode 100644 index 0000000000..0e3161ad41 --- /dev/null +++ b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md @@ -0,0 +1,213 @@ +- 提案名称：`bounded_memory_entry_pagination` +- 开始日期：2026-09-28 +- 跟踪 Issue：[oceanbase/powercontext#1656](https://github.com/oceanbase/powercontext/issues/1656) +- 关联工作：[#1657](https://github.com/oceanbase/powercontext/issues/1657)、 + [#1709](https://github.com/oceanbase/powercontext/pull/1709) 和 + [#1718](https://github.com/oceanbase/powercontext/issues/1718) +- 状态：核心设计原则已达成共识；澄清后的合同已规定；实现尚未开始 + +# 摘要 + +PowerContext 新增一个附加的、工作量有界的 Memory entry 目录查询。现有全量 +`list_memory_entries` 保持不变。新操作按 `entry_id ASC` 返回精确身份和紧凑元数据，不加载 +entry body，并使用签名的不透明 cursor 续页。 + +首页固定一个不可变 Memory revision，后续页在同一 revision 上查询可重建的 revision-valid 目录索引。 +带 tag 过滤的分页还会绑定独立 tag generation；tag 变化后续页显式失效并要求重启。 + +# Part A：SPEC + +## 1. 背景与问题 + +当前 Memory list 路径会解析最新 Artifact，解码完整 manifest，加载所有 entry-version 行，然后才做展示和 tag 过滤。 +即使调用方只需要一小页，数据库、manifest 解码和 entry body 展开工作量仍随整个 Memory 增长。 +只在 HTTP 层切片无法解决这个问题。 + +## 2. 目标行为 + +新增 `POST /v1/memory/entries/query`，operation ID 为 `query_memory_entries`。 + +请求包含 `scope_id`、默认为 `false` 的 `include_inactive`、现有可选 `tag_filter`、默认 50 且范围为 +`1..100` 的 `limit`，以及可选不透明 `cursor`。 + +响应包含固定的 `memory_ref`（Scope 无 Memory 时为 `null`）、按 `entry_id ASC` 排序的 `items` 和 +`next_cursor`。每项只包含精确 citation（`memory_ref`、`entry_id`、`entry_version_id`）、正整数 `version`、 +`kind` 和 `state`；不包含 `text`、`source_refs` 或 `artifact_refs`。body 由现有 exact-detail 操作获取。 + +现有 `list_memory_entries` 的请求、响应、排序和行为均不改变。 + +## 3. 主要场景 + +### 无过滤遍历 + +1. Server 认证调用方并对 Scope 授权。 +2. 首页解析并固定当前 Memory revision；无 Memory 时返回 null reference、空 items 和空 cursor。 +3. 从 page key 之后读取符合条件的紧凑目录行，同时限制 item 数和字节数。 +4. 存在后续行时，签名 cursor 从最后一个已输出 `entry_id` 继续。 +5. 每个后续页都重新授权，并读取同一固定 revision。并发 Memory commit 对本次遍历不可见。 + +### 生命周期与 tag 过滤 + +revision 有效性、lifecycle state 和 tag predicate 必须在 keyset paging 和 item limit 之前应用。 +tag-filtered 首页捕获当前 tag generation；后续页在有界读取前后都验证 generation。有效 Memory-entry tag 变更 +返回 `400 invalid_cursor` + `tag_state_changed`，调用方从首页重启。无操作 replacement 不推进 generation，无 tag cursor +不依赖 generation。这是失效/重启，而不是 tag snapshot 保证。 + +### 未完成索引迁移 + +在认证和授权后，新操作返回 `503 memory_query_index_unavailable`，绝不 fallback 到完整 manifest。 +旧 list、exact detail、search 和 Memory write 在维护窗口外仍可用。运维者对 SQLite/OceanBase 执行显式离线 +`plan`/`apply`/`verify`；只有 verify 成功才标记 ready，apply 中断可恢复且不改写权威 Memory 数据。 +在维护窗口外，即使历史 backfill 尚未完成，新 Memory commit 仍维护自身 directory delta；但在完整 verify 成功前， +新 query 始终不可用。 + +### 页字节上限 + +Server 以固定 4 MiB 预算计量编码后 directory item payload。如果下一项超过 item limit 或字节预算，则在它之前停止。 +合法 v1 字段都已有上限；若损坏数据或未来不兼容形状造成单项无法装入，返回 +`413 memory_directory_item_too_large`，不返回部分页，也不推进 cursor。 + +## 4. 规则与不变量 + +| ID | 规则或不变量 | 违反时结果 | +|---|---|---| +| MEM-PAGE-01 | 新操作是附加能力；旧全量 list 不变。 | 兼容性回归。 | +| MEM-PAGE-02 | 一次遍历固定在一个不可变 Memory revision。 | cursor 无效或实现缺陷。 | +| MEM-PAGE-03 | 先过滤，再按 `entry_id ASC` 分页。 | 遗漏、重复或顺序回归。 | +| MEM-PAGE-04 | 每项保留精确 Memory/entry/entry-version 身份。 | citation 完整性错误。 | +| MEM-PAGE-05 | directory read 不解码 body，不展开 body reference。 | 有界工作量合同失效。 | +| MEM-PAGE-06 | cursor 绑定 endpoint、Scope、filter、limit、order version、revision 和必要的 tag generation。 | `400 invalid_cursor`。 | +| MEM-PAGE-07 | 每页先授权，后暴露私有存在性或 readiness。 | 访问控制缺陷。 | +| MEM-PAGE-08 | 有效 tag 变更原子推进 generation；无操作变更不推进。 | tag 事务回滚。 | +| MEM-PAGE-09 | tag generation 变更只使 filtered continuation 失效。 | `invalid_cursor` + `tag_state_changed`。 | +| MEM-PAGE-10 | query index 是可重建派生状态；Artifact/manifest/version/tag 仍是权威。 | verify 失败。 | +| MEM-PAGE-11 | directory delta 与 Memory 原子提交，且与 changed entry 数成比例。 | 整个事务回滚。 | +| MEM-PAGE-12 | 未 verify 的 index 不服务且不触发无界 fallback。 | `503 memory_query_index_unavailable`。 | +| MEM-PAGE-13 | 每页最多 100 项、4 MiB encoded item payload。 | 在下一项之前结束。 | +| MEM-PAGE-14 | 单个超预算 item 不能产生部分成功。 | `413 memory_directory_item_too_large`。 | + +## 5. 失败、重试和恢复 + +- 格式错误、篡改、跨 endpoint/Scope 或请求不匹配 cursor 使用现有 `400 invalid_cursor`；过期 cursor 使用 + `410 cursor_expired`。 +- `tag_state_changed` 应从首页重启；`memory_query_index_unavailable` 可在 migration verify 后重试。 +- 认证/授权错误优先于 readiness 和 Memory 存在性。 +- oversized item 不返回 item 和替换 cursor。 +- Memory/tag write 与对应 index/generation delta 一起 commit 或 rollback。 +- migration apply 保存有界 checkpoint，可恢复；只有 verify 能标记 complete。 + +## 6. 并发与资源约束 + +- Memory revision 提供稳定跨页 view；tag 不带 revision，因此采用 invalidation，不声称 snapshot。 +- tag query 前后检查 generation；续页边界是最后已输出 `entry_id`，而不是 lookahead row。 +- persistence 每页最多 materialize `limit + 1` 个紧凑 eligible row，不反序列化 body JSON。 +- 增量计量 item encoding，上限 4 MiB；envelope 和 cursor 另外有界。 +- 对 200/1,000/5,000 entry 记录 query plan 和实测工作量；区分 returned/materialized row 与 examined row，不夸大数据库保证。 +- Memory commit 只更新 pointer/state 已变的 validity row；未变 search projection 保持 #1709 的 delta 性质。 +- backfill 使用正 batch size 和持久 checkpoint；startup 不执行无界 rebuild。 + +## 7. 非目标 + +本 PR 不改动 `list_memory_entries`，不在 directory 中返回 body，不实现 #1657 history pagination，不实现 +#1321/#1718 的容量、保留、分裂/路由、压缩或物理删除，不 snapshot 全部 tag，不修改 search/exact-detail/citation 语义， +不让派生行成为权威，也不支持离线 migration 窗口中混用旧/新 writer。 + +## 8. 验收标准 + +| 义务 | 可观察证据 | 层级 | +|---|---|---| +| 附加合同 | 新 OpenAPI operation 存在；旧 schema 兼容。 | 合同/生成测试 | +| 稳定遍历 | 并发写不遗漏、不重复 pinned row。 | Runtime/HTTP 测试 | +| 紧凑精确身份 | item 可由 exact detail 解析，且无 body field。 | Runtime/mapping 测试 | +| 先过滤后分页 | lifecycle/tag 从 eligible row 填页。 | persistence 测试 | +| tag 重启边界 | 有效变更失效；no-op 和 unfiltered cursor 不失效。 | tag/并发测试 | +| 每页授权 | 权限撤销后续页失败且无信息泄露。 | HTTP access 测试 | +| 有界读和响应 | `limit + 1`、无 body decode、count/byte 上限和原子 413。 | instrumented persistence/HTTP 测试 | +| delta write | 只有 changed entry 修改 directory/search projection。 | persistence 测试 | +| 安全 migration | fresh/incomplete/interrupted/resumed/verified/corrupt 状态符合 gate。 | SQLite/OceanBase migration 测试 | +| 规模行为 | 200/1,000/5,000 证据包含 plan、row、byte、write delta 和 batch work。 | 可复现 report | + +# Part B：设计报告 + +## 1. 最终数据流 + +```text +OpenAPI query_memory_entries + -> Server 认证和 Scope 授权 + -> ScopedMemoryApplication.query_directory + -> relational directory authority + -> readiness 和 pinned revision 验证 + -> 可选 tag generation 验证 + -> revision-valid filter + keyset + lookahead query + -> tag generation 再验证 + -> item-byte budget 和 signed continuation + -> Server response mapping +``` + +Memory commit 复用现有事务，比较 previous/current manifest map 后只更新 changed entry 的 validity row。 +Tag replacement 仍由 `RelationalTagService` 拥有；有效 Memory-entry tag delta 在现有 owner-Artifact 串行化下推进专用 generation。 + +## 2. Authority 与 seam + +| 规则或状态 | Authority | Interface | 证据 | +|---|---|---|---| +| wire 形状 | `openapi/powercontext.yaml` | `query_memory_entries` | schema/generation 测试 | +| 授权顺序 | Server application | 现有 Scope authorization | access 测试 | +| directory 语义和 budget | Memory runtime/persistence | `query_directory` | persistence/runtime 测试 | +| cursor 签名和 TTL | `SignedCursorCodec` | exact-context encode/decode | cursor 测试 | +| revision-valid state | relational Memory persistence | directory validity table | database 测试 | +| exact body/version | Memory Artifact 和 entry-version table | 现有 exact read | citation 测试 | +| 可变 tag | `RelationalTagService` 和 tag table | predicate + generation | tag 测试 | +| readiness/checkpoint | query-index migration module | plan/apply/verify | migration 测试 | +| 错误展示 | Server error mapping | 稳定 status/code/details | HTTP 测试 | + +Server 不重新实现 pagination，runtime 不推断 authorization，cursor codec 只认证 context，不拥有 query 语义。 + +## 3. 持久化机制 + +新派生 directory table 按 Scope/Memory Artifact 存储 `entry_id`、精确 `entry_version_id`、`state`、包含起点的 +`valid_from_revision` 和不包含终点的可空 `valid_to_revision`。索引支持受 revision-valid predicate 限制的 +`entry_id` keyset 遍历。`version`/`kind` 从权威 entry-version row 索引 join，不选择或解码 body JSON。 + +在 revision `R`，changed entry 以 `valid_to_revision = R` 关闭前一行，并以 `valid_from_revision = R` +插入新 pointer/state；new entry 只插入；unchanged entry 不写。 + +签名 cursor 仅包含 schema/order version、endpoint、Scope、规范化 filter、limit、pinned Memory Artifact ID/revision、 +可选 tag generation、exclusive `after_entry_id` 和 expiry。 + +小型 tag-generation table 为每个 Scope/Memory Artifact 保存单调 generation，仅在 `RelationalTagService.replace` +对 `memory_entry` 产生有效规范化 delta 时推进。 + +feature-scoped marker 保存 query-index schema version、readiness 和可恢复 checkpoint。新数据库初始为 complete；现有数据库在离线 +verify 确认派生覆盖率和精确身份前保持 incomplete。 + +## 4. 已接受的取舍 + +- body 需要 exact-detail 往返，以换取真正有界的 directory。 +- tag 变化后重启 filtered traversal，不保留无界 snapshot。 +- 现有数据库仅为这一新特性显式离线迁移。 +- revision-valid row 比 current-head table 多占存储，但避免 per-revision manifest copy。 +- 首版排序固定；新排序需要独立版本化 index/cursor 合同。 + +## 5. 义务到测试映射 + +| 义务 | 拥有层 | 测试 | +|---|---|---| +| 公开兼容性 | OpenAPI | schema/generation 和 SDK 测试 | +| revision-valid selection | directory persistence | SQLite/OceanBase keyset 测试 | +| 原子 delta maintenance | Memory commit | rollback 和 row-delta 测试 | +| tag generation | tag persistence | effective/no-op/concurrent mutation 测试 | +| cursor binding | cursor/query seam | mismatch/tamper/expiry 测试 | +| 授权先于暴露 | Server | HTTP/MCP access 测试 | +| byte behavior | query authority/Server | boundary 和 413 测试 | +| readiness | migration module | fresh/incomplete/resume/verify/corrupt 测试 | +| 旧行为不变 | legacy list path | 现有 suite + compatibility regression | +| 资源声称 | persistence | 可复现 scale report | + +## 6. 验证计划与当前状态 + +review 前运行定向 OpenAPI、cursor、Memory persistence、tag、migration、Server、access-control 和 client 测试；SQLite 规模测量； +可用时的 OceanBase 证据；generated-code、contract、docs、format、lint、type check；以及 `make check` 和 `make unit-test`。 +不可用服务和 skipped check 必须与 pass 分开报告。 + +当前状态：本 SPEC 已对照 issue 讨论和当前调用路径审查。功能代码和功能测试尚未实现，不声称任何实现检查已通过。 From 65a27d7fe2a964434816c5c3017b9127f20c5534 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:37:35 +0800 Subject: [PATCH 02/14] feat(memory): track paged directory state --- .../builtin/persistence/memory.py | 56 ++++++++++++- .../builtin/persistence/tables.py | 79 +++++++++++++++++- src/powercontext/builtin/persistence/tags.py | 30 ++++++- tests/builtin/persistence/test_memory.py | 83 ++++++++++++++++++- tests/builtin/persistence/test_records.py | 27 ++++++ 5 files changed, 270 insertions(+), 5 deletions(-) diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index 6e015f5424..c678a938f8 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -22,7 +22,7 @@ from typing import Any, ClassVar from pydantic import RootModel -from sqlalchemy import delete, insert, select +from sqlalchemy import delete, insert, select, update from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactDraft, ArtifactRef @@ -60,8 +60,10 @@ from powercontext.builtin.persistence.memory_index import MemoryIndex, NoMemoryIndex from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, + MEMORY_ENTRY_DIRECTORY_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, ) from powercontext.builtin.persistence.tags import tag_predicate from powercontext.builtin.tags import TagFilter @@ -492,6 +494,58 @@ async def _commit(self, connection: AsyncConnection, value: MemoryCommit) -> Mem insert(MEMORY_ENTRY_VERSIONS_TABLE), [_entry_values(self._scope_id, entry) for entry in value.entry_versions], ) + previous_directory = ( + {} if value.base is None else {item.entry_id: item for item in value.base.content.manifest.entries} + ) + current_directory = {item.entry_id: item for item in value.memory.content.manifest.entries} + directory_changed = tuple( + sorted( + entry_id + for entry_id in previous_directory.keys() | current_directory.keys() + if previous_directory.get(entry_id) != current_directory.get(entry_id) + ) + ) + directory_closed = tuple(entry_id for entry_id in directory_changed if entry_id in previous_directory) + if directory_closed: + await connection.execute( + update(MEMORY_ENTRY_DIRECTORY_TABLE) + .where( + MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == self._scope_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == value.memory.artifact_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id.in_(directory_closed), + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_to_revision.is_(None), + ) + .values(valid_to_revision=value.memory.revision) + ) + directory_inserted = tuple( + current_directory[entry_id] for entry_id in directory_changed if entry_id in current_directory + ) + if directory_inserted: + await connection.execute( + insert(MEMORY_ENTRY_DIRECTORY_TABLE), + [ + { + "scope_id": self._scope_id, + "family": Memory.family, + "memory_artifact_id": value.memory.artifact_id, + "entry_id": item.entry_id, + "entry_version_id": item.entry_version_id, + "state": item.state, + "valid_from_revision": value.memory.revision, + "valid_to_revision": None, + } + for item in directory_inserted + ], + ) + if value.base is None: + await connection.execute( + insert(MEMORY_TAG_GENERATIONS_TABLE).values( + scope_id=self._scope_id, + family=Memory.family, + memory_artifact_id=value.memory.artifact_id, + generation=0, + ) + ) # Only entries whose pointer or state changed need projection work; the # rest of the active head stays exactly as the previous revision left it. previous_active = ( diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 530d3f13de..28f1d6045c 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1013,7 +1013,84 @@ def _entry_text_type(): ) -MEMORY_TABLES = (MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_HEADS_TABLE) +MEMORY_ENTRY_DIRECTORY_TABLE = Table( + "pc_memory_entry_directory", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("family", identity_string(MAX_ARTIFACT_FAMILY_LENGTH), nullable=False), + Column("memory_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("entry_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), primary_key=True), + Column("entry_version_id", identity_string(MAX_MEMORY_ENTRY_ID_LENGTH), nullable=False), + Column("state", identity_string(8), nullable=False), + Column("valid_from_revision", Integer, primary_key=True), + Column("valid_to_revision", Integer), + ForeignKeyConstraint( + ("scope_id", "family", "memory_artifact_id", "valid_from_revision"), + ( + "pc_artifacts.scope_id", + "pc_artifacts.family", + "pc_artifacts.artifact_id", + "pc_artifacts.revision", + ), + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ("scope_id", "memory_artifact_id", "entry_id", "entry_version_id"), + ( + "pc_memory_entry_versions.scope_id", + "pc_memory_entry_versions.memory_artifact_id", + "pc_memory_entry_versions.entry_id", + "pc_memory_entry_versions.entry_version_id", + ), + ondelete="RESTRICT", + ), + CheckConstraint("state IN ('active', 'inactive')", name="ck_pc_memory_entry_directory_state"), + CheckConstraint( + "valid_from_revision > 0", + name="ck_pc_memory_entry_directory_from_positive", + ), + CheckConstraint( + "valid_to_revision IS NULL OR valid_to_revision > valid_from_revision", + name="ck_pc_memory_entry_directory_valid_range", + ), + Index( + "ix_pc_memory_entry_directory_page", + "scope_id", + "memory_artifact_id", + "entry_id", + "valid_from_revision", + "valid_to_revision", + "state", + ), +) + + +MEMORY_TAG_GENERATIONS_TABLE = Table( + "pc_memory_tag_generations", + SHARED_METADATA, + Column("scope_id", identity_string(MAX_SCOPE_ID_LENGTH), primary_key=True), + Column("family", identity_string(MAX_ARTIFACT_FAMILY_LENGTH), nullable=False), + Column("memory_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH), primary_key=True), + Column("generation", BigInteger, nullable=False), + ForeignKeyConstraint( + ("scope_id", "family", "memory_artifact_id"), + ( + "pc_artifact_heads.scope_id", + "pc_artifact_heads.family", + "pc_artifact_heads.artifact_id", + ), + ondelete="CASCADE", + ), + CheckConstraint("generation >= 0", name="ck_pc_memory_tag_generations_nonnegative"), +) + + +MEMORY_TABLES = ( + MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, +) TOPIC_MEMORY_TABLES = ( TOPIC_MEMORY_RETRIEVAL_SHAPE_TABLE, diff --git a/src/powercontext/builtin/persistence/tags.py b/src/powercontext/builtin/persistence/tags.py index 7036dd1222..5cbff35e83 100644 --- a/src/powercontext/builtin/persistence/tags.py +++ b/src/powercontext/builtin/persistence/tags.py @@ -35,7 +35,11 @@ from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import RepositoryNotFoundError -from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACT_TAGS_TABLE +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACT_TAGS_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, +) from powercontext.builtin.records import BaseValueNotFoundError, CursorExpiredError, InvalidCursorError from powercontext.builtin.tags import ( ArtifactTagSet, @@ -202,6 +206,30 @@ async def replace( .where(_where(identity), ARTIFACT_TAGS_TABLE.c.tag_key == key) .values(tag=label) ) + if isinstance(target, MemoryEntryTagTarget) and previous != desired: + generation = await connection.scalar( + select(MEMORY_TAG_GENERATIONS_TABLE.c.generation).where( + MEMORY_TAG_GENERATIONS_TABLE.c.scope_id == scope_id, + MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id == target.artifact_id, + ) + ) + if generation is None: + await connection.execute( + insert(MEMORY_TAG_GENERATIONS_TABLE).values( + scope_id=scope_id, + family=target.family, + memory_artifact_id=target.artifact_id, + generation=0, + ) + ) + await connection.execute( + update(MEMORY_TAG_GENERATIONS_TABLE) + .where( + MEMORY_TAG_GENERATIONS_TABLE.c.scope_id == scope_id, + MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id == target.artifact_id, + ) + .values(generation=MEMORY_TAG_GENERATIONS_TABLE.c.generation + 1) + ) return tag_set(scope_id, target, tags) async def query(self, scope_id: str, query: TagQuery, *, caller: str = "runtime") -> TagQueryPage: diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index 86a96b8b86..0526555981 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -16,15 +16,17 @@ import asyncio -from sqlalchemy import BigInteger, DateTime, Integer, String +from sqlalchemy import BigInteger, DateTime, Integer, String, select from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint from powercontext.builtin.artifacts.memory import MemoryEntryInput from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( + MEMORY_ENTRY_DIRECTORY_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.sources import ContentCapture, ContentSource @@ -53,17 +55,27 @@ def _key_budget(constraint: PrimaryKeyConstraint | UniqueConstraint | ForeignKey def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() -> None: dialect = mysql.dialect() versions = str(CreateTable(MEMORY_ENTRY_VERSIONS_TABLE).compile(dialect=dialect)) + directory = str(CreateTable(MEMORY_ENTRY_DIRECTORY_TABLE).compile(dialect=dialect)) heads = str(CreateTable(MEMORY_ENTRY_HEADS_TABLE).compile(dialect=dialect)) + generations = str(CreateTable(MEMORY_TAG_GENERATIONS_TABLE).compile(dialect=dialect)) assert "scope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in versions assert "text MEDIUMTEXT NOT NULL" in versions assert "source_refs MEDIUMBLOB NOT NULL" in versions assert "artifact_refs MEDIUMBLOB NOT NULL" in versions + assert "valid_from_revision INTEGER NOT NULL" in directory + assert "valid_to_revision INTEGER" in directory assert "searchable_text MEDIUMTEXT NOT NULL" in heads + assert "generation BIGINT NOT NULL" in generations budgets = [ _key_budget(constraint) - for table in (MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_HEADS_TABLE) + for table in ( + MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, + ) for constraint in table.constraints if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) ] @@ -72,6 +84,73 @@ def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() assert all(budget < _INNODB_MAX_INDEX_BYTES for budget in budgets) +def test_sqlite_memory_directory_records_only_changed_revision_intervals() -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + service = context.artifacts.memory + first = await service.remember( + memory=None, + entries=( + MemoryEntryInput(kind="decision", text="Keep the stable entry unchanged."), + MemoryEntryInput(kind="constraint", text="Revise and then retire this entry."), + ), + mode="append", + ) + assert first is not None + stable, changing = await service.entries(first) + second = await service.remember( + memory=first, + entries=( + MemoryEntryInput( + entry=changing, + kind=changing.kind, + text="This entry now has a second immutable version.", + ), + ), + mode="append", + ) + assert second is not None + revised = next(entry for entry in await service.entries(second) if entry.entry_id == changing.entry_id) + third = await service.forget(second, entries=(revised,), reason="No longer current.") + + async with contexts.database.transaction() as connection: + rows = ( + ( + await connection.execute( + select(MEMORY_ENTRY_DIRECTORY_TABLE).order_by( + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision, + ) + ) + ) + .mappings() + .all() + ) + generation = await connection.scalar(select(MEMORY_TAG_GENERATIONS_TABLE.c.generation)) + + by_entry = { + entry_id: [row for row in rows if row["entry_id"] == entry_id] + for entry_id in {stable.entry_id, changing.entry_id} + } + assert [ + (row["entry_version_id"], row["state"], row["valid_from_revision"], row["valid_to_revision"]) + for row in by_entry[stable.entry_id] + ] == [(stable.entry_version_id, "active", 1, None)] + assert [ + (row["entry_version_id"], row["state"], row["valid_from_revision"], row["valid_to_revision"]) + for row in by_entry[changing.entry_id] + ] == [ + (changing.entry_version_id, "active", 1, 2), + (revised.entry_version_id, "active", 2, 3), + (revised.entry_version_id, "inactive", 3, None), + ] + assert third.revision == 3 + assert generation == 0 + + asyncio.run(scenario()) + + def test_sqlite_memory_backend_commits_authoritative_history_and_fts() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: diff --git a/tests/builtin/persistence/test_records.py b/tests/builtin/persistence/test_records.py index 91f1d8fc8c..08c9bf519c 100644 --- a/tests/builtin/persistence/test_records.py +++ b/tests/builtin/persistence/test_records.py @@ -50,6 +50,7 @@ BUILTIN_TABLES, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, SKILL_PACKAGES_TABLE, SOURCES_TABLE, ) @@ -243,6 +244,32 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_memory_entry_tag_generation_advances_only_for_effective_changes() -> None: + async def scenario() -> None: + async with SQLiteProfile.open(SQLiteConfig(), tables=BUILTIN_TABLES) as profile: + records, _, _ = _services(profile) + created = await records.create_artifact("scope-a", "memory", ArtifactWrite(content=_memory_content())) + head = await records.get_artifact("scope-a", "memory", created.artifact_id) + entry_id = MemoryContent.model_validate(head.content).manifest.entries[0].entry_id + target = MemoryEntryTagTarget(artifact_id=created.artifact_id, entry_id=entry_id) + + empty = await records.get_tags("scope-a", target) + tagged = await records.replace_tags("scope-a", target, ("Stable",), expected_etag=empty.etag) + unchanged = await records.replace_tags("scope-a", target, ("Stable",), expected_etag=tagged.etag) + await records.replace_tags("scope-a", target, ("stable",), expected_etag=unchanged.etag) + + async with profile.database.transaction() as connection: + generation = await connection.scalar( + select(MEMORY_TAG_GENERATIONS_TABLE.c.generation).where( + MEMORY_TAG_GENERATIONS_TABLE.c.scope_id == "scope-a", + MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id == created.artifact_id, + ) + ) + assert generation == 2 + + asyncio.run(scenario()) + + def _handoff_content(objective: str = "Transfer the API test result.") -> dict[str, JsonValue]: return { "schema": "powercontext.handoff.v1", From a5b5a7e37e796cc05949aba696e46f6aced3afb1 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:33:43 +0800 Subject: [PATCH 03/14] feat(memory): query bounded entry directory --- .../builtin/artifacts/memory/__init__.py | 8 + .../builtin/artifacts/memory/errors.py | 4 + .../builtin/artifacts/memory/protocols.py | 33 +++ .../builtin/artifacts/memory/service.py | 7 + .../builtin/persistence/memory.py | 258 +++++++++++++++++- src/powercontext/builtin/runtime/__init__.py | 5 + .../builtin/runtime/application.py | 15 + src/powercontext/builtin/runtime/models.py | 9 + .../builtin/runtime/relational.py | 10 +- tests/builtin/persistence/test_memory.py | 118 +++++++- 10 files changed, 461 insertions(+), 6 deletions(-) diff --git a/src/powercontext/builtin/artifacts/memory/__init__.py b/src/powercontext/builtin/artifacts/memory/__init__.py index ed1ce9e4e7..ec7594d76f 100644 --- a/src/powercontext/builtin/artifacts/memory/__init__.py +++ b/src/powercontext/builtin/artifacts/memory/__init__.py @@ -21,6 +21,7 @@ InvalidMemoryCitationError, InvalidMemoryEvidenceError, MemoryBackendConfigurationError, + MemoryDirectoryItemTooLargeError, MemoryEntryInactiveError, MemoryEntryNotFoundError, MemoryLayerError, @@ -74,6 +75,9 @@ MemoryBackend, MemoryCandidateRequest, MemoryCommit, + MemoryDirectoryItem, + MemoryDirectoryPage, + MemoryDirectoryQuery, MemoryProjection, MemorySearchChannels, MemorySearchRequest, @@ -122,6 +126,10 @@ "MemoryCitation", "MemoryCommit", "MemoryContent", + "MemoryDirectoryItem", + "MemoryDirectoryItemTooLargeError", + "MemoryDirectoryPage", + "MemoryDirectoryQuery", "MemoryEntryInactiveError", "MemoryEntryInput", "MemoryEntryNotFoundError", diff --git a/src/powercontext/builtin/artifacts/memory/errors.py b/src/powercontext/builtin/artifacts/memory/errors.py index e7b61d1070..2c954af269 100644 --- a/src/powercontext/builtin/artifacts/memory/errors.py +++ b/src/powercontext/builtin/artifacts/memory/errors.py @@ -104,3 +104,7 @@ def __init__(self, code: str) -> None: class MemoryBackendConfigurationError(MemoryLayerError, RuntimeError): """Raised when a repository cannot satisfy its declared configuration.""" + + +class MemoryDirectoryItemTooLargeError(MemoryLayerError): + """Raised when one directory item cannot fit the public page budget.""" diff --git a/src/powercontext/builtin/artifacts/memory/protocols.py b/src/powercontext/builtin/artifacts/memory/protocols.py index c79303fcb0..d54a86a34f 100644 --- a/src/powercontext/builtin/artifacts/memory/protocols.py +++ b/src/powercontext/builtin/artifacts/memory/protocols.py @@ -27,7 +27,9 @@ Memory, MemoryCapabilities, MemoryChannelHit, + MemoryCitation, MemoryEntryInput, + MemoryEntryState, MemoryEntryVersion, MemoryHit, MemoryRevisionChanges, @@ -92,6 +94,32 @@ class MemorySearchChannels(BaseModel): vector: tuple[MemoryChannelHit, ...] = () +class MemoryDirectoryQuery(BaseModel): + """A bounded directory traversal over one stable Memory identity.""" + + include_inactive: bool = False + tag_filter: TagFilter | None = None + limit: int = 50 + cursor: str | None = None + + +class MemoryDirectoryItem(BaseModel): + """Compact exact identity for one entry without its body or evidence.""" + + citation: MemoryCitation + version: int + kind: str + state: MemoryEntryState + + +class MemoryDirectoryPage(BaseModel): + """One bounded page pinned to an exact Memory Revision.""" + + memory_ref: ArtifactRef | None + items: tuple[MemoryDirectoryItem, ...] = () + next_cursor: str | None = None + + class CandidatePipeline(Protocol): """Produce untrusted Memory candidates from canonical bounded evidence.""" @@ -133,6 +161,11 @@ async def latest(self, artifact_id: str, /) -> Memory: ... + async def query_directory(self, artifact_id: str, query: MemoryDirectoryQuery, /) -> MemoryDirectoryPage: + """Read one compact keyset page without loading the Memory manifest or entry bodies.""" + + ... + async def entries(self, memory: ArtifactRef, /) -> tuple[MemoryEntryVersion, ...]: """Load the entry versions referenced by an exact manifest.""" diff --git a/src/powercontext/builtin/artifacts/memory/service.py b/src/powercontext/builtin/artifacts/memory/service.py index 84f0541e24..4940b7d2fe 100644 --- a/src/powercontext/builtin/artifacts/memory/service.py +++ b/src/powercontext/builtin/artifacts/memory/service.py @@ -76,6 +76,8 @@ MemoryBackend, MemoryCandidateRequest, MemoryCommit, + MemoryDirectoryPage, + MemoryDirectoryQuery, MemoryProjection, MemorySearchRequest, MemoryWritePlan, @@ -197,6 +199,11 @@ async def latest(self, memory: Memory, /) -> Memory: canonical = await self.get(memory) return await self._backend.latest(canonical.artifact_id) + async def query_directory(self, artifact_id: str, query: MemoryDirectoryQuery, /) -> MemoryDirectoryPage: + """Read compact entry identities without expanding the authoritative manifest.""" + + return await self._backend.query_directory(artifact_id, query) + async def revisions(self, memory: Memory, /) -> tuple[Memory, ...]: """Return the visible Memory history in ascending Revision order.""" diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index c678a938f8..36a10cdde9 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -16,13 +16,15 @@ from __future__ import annotations +import json from collections.abc import AsyncIterator, Mapping from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, replace from typing import Any, ClassVar -from pydantic import RootModel -from sqlalchemy import delete, insert, select, update +import rfc8785 +from pydantic import JsonValue, RootModel +from sqlalchemy import delete, insert, or_, select, update from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts import ArtifactDraft, ArtifactRef @@ -33,6 +35,10 @@ MemoryCapabilities, MemoryCommit, MemoryContent, + MemoryDirectoryItem, + MemoryDirectoryItemTooLargeError, + MemoryDirectoryPage, + MemoryDirectoryQuery, MemoryEntryVersion, MemoryHit, MemoryProjection, @@ -55,6 +61,7 @@ from powercontext.builtin.inference import EmbeddingModel from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.codec import dump_model, load_model, stored_bytes +from powercontext.builtin.persistence.cursor_codec import Clock, SignedCursorCodec from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import RepositoryNotFoundError from powercontext.builtin.persistence.memory_index import MemoryIndex, NoMemoryIndex @@ -66,10 +73,13 @@ MEMORY_TAG_GENERATIONS_TABLE, ) from powercontext.builtin.persistence.tags import tag_predicate +from powercontext.builtin.records import InvalidBaseAccessRequestError, InvalidCursorError from powercontext.builtin.tags import TagFilter from powercontext.errors import ArtifactNotFoundError from powercontext.sources import SourceRef +_MEMORY_DIRECTORY_PAGE_BUDGET_BYTES = 4 * 1024 * 1024 + class _SourceRefs(RootModel[tuple[SourceRef, ...]]): pass @@ -114,6 +124,15 @@ def __init__(self, code: str) -> None: super().__init__(messages[code]) +class _MemoryDirectoryConfigurationError(MemoryBackendConfigurationError): + def __init__(self, code: str) -> None: + messages = { + "revision": "directory query did not pin a Memory revision", + "tag-generation": "Memory tag generation is unavailable", + } + super().__init__(messages[code]) + + class RelationalMemoryBackend: """Use shared Artifact revisions plus Memory-owned entry projections.""" @@ -125,12 +144,20 @@ def __init__( artifacts: ArtifactRepository, index: MemoryIndex | None = None, connection: AsyncConnection | None = None, + cursor_secret: bytes | None = None, + cursor_clock: Clock | None = None, + cursor_ttl_seconds: int = 3_600, ) -> None: self._database = database self._scope_id = scope_id self._artifacts = artifacts self._index = NoMemoryIndex() if index is None else index self._bound_connection = connection + self._cursor_codec = SignedCursorCodec( + secret=cursor_secret, + clock=cursor_clock, + ttl_seconds=cursor_ttl_seconds, + ) async def capabilities(self) -> MemoryCapabilities: return self._index.capabilities @@ -158,6 +185,93 @@ async def latest(self, artifact_id: str, /) -> Memory: raise ArtifactNotFoundError(artifact_id) from None return _require_memory(artifact) + async def query_directory(self, artifact_id: str, query: MemoryDirectoryQuery, /) -> MemoryDirectoryPage: + if isinstance(query.limit, bool) or not 1 <= query.limit <= 100: + raise InvalidBaseAccessRequestError("limit", "must be between 1 and 100") + expected_cursor = _directory_cursor_context(self._scope_id, artifact_id, query) + cursor_state = self._cursor_codec.after_text(query.cursor, expected_cursor) + pinned_revision, expected_tag_generation, after_entry_id = _decode_directory_cursor(cursor_state) + selected = await self._select_directory_rows( + artifact_id, + query, + pinned_revision=pinned_revision, + expected_tag_generation=expected_tag_generation, + after_entry_id=after_entry_id, + ) + if selected is None: + return MemoryDirectoryPage(memory_ref=None) + pinned_revision, expected_tag_generation, rows = selected + memory_ref = ArtifactRef(family=Memory.family, artifact_id=artifact_id, revision=pinned_revision) + items, has_more = _directory_items(memory_ref, rows, query.limit) + next_cursor = None + if has_more and items: + cursor_position = rfc8785.dumps([ + pinned_revision, + expected_tag_generation, + items[-1].citation.entry_id, + ]).decode("utf-8") + next_cursor = self._cursor_codec.encode(expected_cursor, cursor_position) + return MemoryDirectoryPage(memory_ref=memory_ref, items=items, next_cursor=next_cursor) + + async def _select_directory_rows( + self, + artifact_id: str, + query: MemoryDirectoryQuery, + *, + pinned_revision: int | None, + expected_tag_generation: int | None, + after_entry_id: str, + ) -> tuple[int, int | None, list[Mapping[Any, Any]]] | None: + async with self._database.transaction() as connection: + current_revision = await connection.scalar( + select(ARTIFACT_HEADS_TABLE.c.revision).where( + ARTIFACT_HEADS_TABLE.c.scope_id == self._scope_id, + ARTIFACT_HEADS_TABLE.c.family == Memory.family, + ARTIFACT_HEADS_TABLE.c.artifact_id == artifact_id, + ) + ) + if current_revision is None: + if query.cursor is not None: + raise InvalidCursorError + return None + current_revision = int(current_revision) + if query.cursor is None: + pinned_revision = current_revision + elif pinned_revision is None or not 1 <= pinned_revision <= current_revision: + raise InvalidCursorError + + tag_generation = await _memory_tag_generation( + connection, + self._scope_id, + artifact_id, + required=query.tag_filter is not None, + ) + if query.cursor is None: + expected_tag_generation = tag_generation + elif expected_tag_generation != tag_generation: + raise InvalidCursorError("tag_state_changed") + + statement = _directory_statement( + self._scope_id, + artifact_id, + query, + pinned_revision, + after_entry_id, + ) + rows = (await connection.execute(statement)).mappings().all() + if query.tag_filter is not None and query.cursor is not None: + observed_generation = await _memory_tag_generation( + connection, + self._scope_id, + artifact_id, + required=True, + ) + if observed_generation != expected_tag_generation: + raise InvalidCursorError("tag_state_changed") + if pinned_revision is None: + raise _MemoryDirectoryConfigurationError("revision") + return pinned_revision, expected_tag_generation, list(rows) + async def tagged_entry_ids(self, memory: ArtifactRef, tag_filter: TagFilter) -> frozenset[str]: canonical = await self.get(memory) versions = tuple(entry.entry_version_id for entry in canonical.content.manifest.entries) @@ -637,6 +751,146 @@ def _validate_commit(value: MemoryCommit) -> None: raise _InvalidMemoryCommitError("projection") +def _directory_cursor_context( + scope_id: str, + artifact_id: str, + query: MemoryDirectoryQuery, +) -> dict[str, JsonValue]: + return { + "version": 1, + "endpoint": "query_memory_entries", + "scope_id": scope_id, + "memory_artifact_id": artifact_id, + "include_inactive": query.include_inactive, + "tags": [] if query.tag_filter is None else list(query.tag_filter.keys), + "tag_match": None if query.tag_filter is None else query.tag_filter.match, + "limit": query.limit, + "order": "entry_id:asc", + } + + +def _directory_statement( + scope_id: str, + artifact_id: str, + query: MemoryDirectoryQuery, + pinned_revision: int, + after_entry_id: str, +): + directory = MEMORY_ENTRY_DIRECTORY_TABLE + versions = MEMORY_ENTRY_VERSIONS_TABLE + statement = ( + select( + directory.c.entry_id, + directory.c.entry_version_id, + directory.c.state, + versions.c.version, + versions.c.kind, + ) + .join( + versions, + (versions.c.scope_id == directory.c.scope_id) + & (versions.c.memory_artifact_id == directory.c.memory_artifact_id) + & (versions.c.entry_id == directory.c.entry_id) + & (versions.c.entry_version_id == directory.c.entry_version_id), + ) + .where( + directory.c.scope_id == scope_id, + directory.c.memory_artifact_id == artifact_id, + directory.c.valid_from_revision <= pinned_revision, + or_( + directory.c.valid_to_revision.is_(None), + directory.c.valid_to_revision > pinned_revision, + ), + directory.c.entry_id > after_entry_id, + ) + .order_by(directory.c.entry_id) + .limit(query.limit + 1) + ) + if not query.include_inactive: + statement = statement.where(directory.c.state == "active") + if query.tag_filter is not None: + statement = statement.where( + tag_predicate( + scope_id, + directory.c.family, + directory.c.memory_artifact_id, + "memory_entry", + directory.c.entry_id, + query.tag_filter, + ) + ) + return statement + + +def _directory_items( + memory_ref: ArtifactRef, + rows: list[Mapping[Any, Any]], + limit: int, +) -> tuple[tuple[MemoryDirectoryItem, ...], bool]: + items: list[MemoryDirectoryItem] = [] + page_bytes = 0 + for row in rows: + if len(items) == limit: + return tuple(items), True + item = MemoryDirectoryItem.model_validate({ + "citation": { + "memory_ref": memory_ref, + "entry_id": row["entry_id"], + "entry_version_id": row["entry_version_id"], + }, + "version": row["version"], + "kind": row["kind"], + "state": row["state"], + }) + encoded_bytes = len(item.model_dump_json().encode("utf-8")) + if page_bytes + encoded_bytes > _MEMORY_DIRECTORY_PAGE_BUDGET_BYTES: + if not items: + raise MemoryDirectoryItemTooLargeError + return tuple(items), True + items.append(item) + page_bytes += encoded_bytes + return tuple(items), False + + +def _decode_directory_cursor(value: str) -> tuple[int | None, int | None, str]: + if not value: + return None, None, "" + try: + state = json.loads(value) + except (TypeError, ValueError): + raise InvalidCursorError from None + if not isinstance(state, list) or len(state) != 3: + raise InvalidCursorError + revision, generation, after_entry_id = state + if not isinstance(revision, int) or isinstance(revision, bool) or revision < 1: + raise InvalidCursorError + if generation is not None and (not isinstance(generation, int) or isinstance(generation, bool) or generation < 0): + raise InvalidCursorError + if not isinstance(after_entry_id, str) or not after_entry_id: + raise InvalidCursorError + return revision, generation, after_entry_id + + +async def _memory_tag_generation( + connection: AsyncConnection, + scope_id: str, + artifact_id: str, + *, + required: bool, +) -> int | None: + if not required: + return None + value = await connection.scalar( + select(MEMORY_TAG_GENERATIONS_TABLE.c.generation).where( + MEMORY_TAG_GENERATIONS_TABLE.c.scope_id == scope_id, + MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id == artifact_id, + ) + ) + if value is None: + raise _MemoryDirectoryConfigurationError("tag-generation") + return int(value) + + def _entry_values(scope_id: str, value: MemoryEntryVersion) -> dict[str, object]: return { "scope_id": scope_id, diff --git a/src/powercontext/builtin/runtime/__init__.py b/src/powercontext/builtin/runtime/__init__.py index 5953e26801..5aca288c9c 100644 --- a/src/powercontext/builtin/runtime/__init__.py +++ b/src/powercontext/builtin/runtime/__init__.py @@ -34,6 +34,7 @@ ) from powercontext.builtin.artifacts.memory.models import MemoryChange from powercontext.builtin.artifacts.memory.prompts import MemoryExtractionProfile +from powercontext.builtin.artifacts.memory.protocols import MemoryDirectoryItem, MemoryDirectoryPage from powercontext.builtin.dream.application import DreamApplication, ScopedDreamApplication from powercontext.builtin.dream.models import ( CreateDreamRunRequest, @@ -146,6 +147,7 @@ PreparedContext, ProposeExperienceRequest, ProposeSkillRequest, + QueryMemoryEntriesRequest, RejectArtifactCandidateRequest, RememberMemoryRequest, ResolveExternalSkillRequest, @@ -278,6 +280,8 @@ "MemoryChange", "MemoryChangesPage", "MemoryCitation", + "MemoryDirectoryItem", + "MemoryDirectoryPage", "MemoryEntriesPage", "MemoryEntryInput", "MemoryEntryInventoryStatistics", @@ -304,6 +308,7 @@ "PreparedHandoff", "ProposeExperienceRequest", "ProposeSkillRequest", + "QueryMemoryEntriesRequest", "ReadinessCheckStatus", "ReadinessProbeDefinition", "RecallTokenDay", diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index a734414545..f502415cce 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -57,6 +57,8 @@ EmbeddingProfile, Memory, MemoryCitation, + MemoryDirectoryPage, + MemoryDirectoryQuery, MemoryEntryInput, MemoryEntryVersion, MemoryHit, @@ -192,6 +194,7 @@ PreparedContext, ProposeExperienceRequest, ProposeSkillRequest, + QueryMemoryEntriesRequest, RejectArtifactCandidateRequest, RememberMemoryRequest, ResolveExternalSkillRequest, @@ -2458,6 +2461,18 @@ async def search(self, request: SearchMemoryRequest, /) -> MemorySearchPage: rerank=result.rerank, ) + async def query(self, request: QueryMemoryEntriesRequest, /) -> MemoryDirectoryPage: + async with self._runtime._context(self.scope_id) as context: + return await context.artifacts.memory.query_directory( + context.artifacts.memory_artifact_id, + MemoryDirectoryQuery( + include_inactive=request.include_inactive, + tag_filter=request.tag_filter, + limit=request.limit, + cursor=request.cursor, + ), + ) + async def list(self, *, include_inactive: bool = False, tag_filter: TagFilter | None = None) -> MemoryEntriesPage: async with self._runtime._context(self.scope_id) as context: service = context.artifacts.memory diff --git a/src/powercontext/builtin/runtime/models.py b/src/powercontext/builtin/runtime/models.py index d7f37cf654..8ba1aa6e79 100644 --- a/src/powercontext/builtin/runtime/models.py +++ b/src/powercontext/builtin/runtime/models.py @@ -180,6 +180,15 @@ class SearchMemoryRequest(BaseModel): tag_filter: TagFilter | None = None +class QueryMemoryEntriesRequest(BaseModel): + """Traverse compact entry identities without loading entry bodies.""" + + include_inactive: bool = Field(default=False, strict=True) + tag_filter: TagFilter | None = None + limit: Annotated[int, Field(ge=1, le=100)] = 50 + cursor: Annotated[str | None, Field(max_length=4096)] = None + + class MemorySearchPage(BaseModel): """Search results that can represent a scope with no Memory.""" diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 0ed72c576c..4cccdb710b 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -307,6 +307,7 @@ class _ScopedServices: source_registry: SourceDefinitionRegistry prompts: PromptService generation_receipts: HandoffGenerationReceipts + cursor_secret: bytes def generation_sources(self) -> GenerationSourceAccess: return GenerationSourceAccess(self.repositories.sources) @@ -339,6 +340,7 @@ def memory( artifacts=self.repositories.artifacts, index=self.index, connection=connection, + cursor_secret=self.cursor_secret, ), candidate_pipeline=self.candidate_pipeline, embedding_model=self.embedding_model, @@ -527,7 +529,8 @@ def __init__( topic_memory_write_concurrency: int = 4, ) -> None: self.database = database - self.scopes = ScopeApplication(database, cursor_secret=cursor_secret) + self._cursor_secret = cursor_secret if cursor_secret is not None else secrets.token_bytes(32) + self.scopes = ScopeApplication(database, cursor_secret=self._cursor_secret) self.source_registry = source_registry or BUILTIN_SOURCE_REGISTRY self.index = NoMemoryIndex() if index is None else index self.topic_memory_index = NoTopicMemoryIndex() if topic_memory_index is None else topic_memory_index @@ -579,7 +582,7 @@ def __init__( ) self.prompts = PromptService(self.prompt_registry, self._prompt_head, prompt_demonstrators) self._generation_receipts = HandoffGenerationReceipts( - cursor_secret if cursor_secret is not None else secrets.token_bytes(32), + self._cursor_secret, verification_keys=handoff_verification_keys, ) topic_memory_writer = TopicMemoryManagementWriter( @@ -631,7 +634,7 @@ def __init__( self.repositories.artifacts, family_writers, id_factory=id_factory, - cursor_secret=cursor_secret, + cursor_secret=self._cursor_secret, processing_pending=self.repositories.processing_pending, source_processing_bindings=(TOPIC_MEMORY_SOURCE_WINDOW_BINDING,), topic_memory_list_reader=TopicMemoryArtifactListReader( @@ -1407,6 +1410,7 @@ def _services_for(self, scope_id: str) -> _ScopedServices: source_lock=self._source_locks.setdefault(scope, asyncio.Lock()), prompts=self.prompts, generation_receipts=self._generation_receipts, + cursor_secret=self._cursor_secret, token_estimator=self._token_estimator, source_registry=self.source_registry, ) diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index 0526555981..9c61485d06 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -16,11 +16,12 @@ import asyncio +import pytest from sqlalchemy import BigInteger, DateTime, Integer, String, select from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable, ForeignKeyConstraint, PrimaryKeyConstraint, UniqueConstraint -from powercontext.builtin.artifacts.memory import MemoryEntryInput +from powercontext.builtin.artifacts.memory import MemoryDirectoryQuery, MemoryEntryInput from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.persistence.tables import ( MEMORY_ENTRY_DIRECTORY_TABLE, @@ -28,8 +29,10 @@ MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_TAG_GENERATIONS_TABLE, ) +from powercontext.builtin.records import InvalidCursorError from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts from powercontext.builtin.sources import ContentCapture, ContentSource +from powercontext.builtin.tags import MemoryEntryTagTarget, TagFilter _INNODB_MAX_INDEX_BYTES = 3072 @@ -151,6 +154,119 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_sqlite_memory_directory_query_pins_revision_and_invalidates_changed_tags(monkeypatch) -> None: + async def scenario() -> None: + async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: + context = await contexts.get("project") + service = context.artifacts.memory + first = await service.remember( + memory=None, + entries=tuple(MemoryEntryInput(kind="fact", text=f"Directory body {index}") for index in range(3)), + mode="append", + ) + assert first is not None + original_ids = sorted(item.entry_id for item in await service.entries(first)) + + import powercontext.builtin.persistence.memory as persistence_memory + + decoded = 0 + original_decode = persistence_memory._decode_entry + + def counting_decode(row): + nonlocal decoded + decoded += 1 + return original_decode(row) + + monkeypatch.setattr(persistence_memory, "_decode_entry", counting_decode) + page_one = await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery(limit=1), + ) + assert page_one.memory_ref == first.as_ref() + assert [item.citation.entry_id for item in page_one.items] == original_ids[:1] + assert set(page_one.items[0].model_dump()) == {"citation", "version", "kind", "state"} + assert page_one.next_cursor is not None + assert decoded == 0 + + second = await service.remember( + memory=first, + entries=(MemoryEntryInput(kind="fact", text="Visible only to a new traversal."),), + mode="append", + ) + assert second is not None + remaining = [] + cursor = page_one.next_cursor + while cursor is not None: + page = await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery(limit=1, cursor=cursor), + ) + assert page.memory_ref == first.as_ref() + remaining.extend(item.citation.entry_id for item in page.items) + cursor = page.next_cursor + assert remaining == original_ids[1:] + assert decoded == 0 + with pytest.raises(InvalidCursorError): + await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery(limit=2, cursor=page_one.next_cursor), + ) + + tagged_ids = sorted(original_ids[:2]) + for entry_id in tagged_ids: + target = MemoryEntryTagTarget(artifact_id=first.artifact_id, entry_id=entry_id) + empty = await contexts.records.get_tags("project", target) + await contexts.records.replace_tags( + "project", + target, + ("paged",), + expected_etag=empty.etag, + ) + filtered = await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery(tag_filter=TagFilter(tags=("PAGED",)), limit=1), + ) + assert [item.citation.entry_id for item in filtered.items] == tagged_ids[:1] + assert filtered.next_cursor is not None + + unchanged_target = MemoryEntryTagTarget(artifact_id=first.artifact_id, entry_id=tagged_ids[0]) + unchanged = await contexts.records.get_tags("project", unchanged_target) + await contexts.records.replace_tags( + "project", + unchanged_target, + unchanged.tags, + expected_etag=unchanged.etag, + ) + continued = await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery( + tag_filter=TagFilter(tags=("paged",)), + limit=1, + cursor=filtered.next_cursor, + ), + ) + assert [item.citation.entry_id for item in continued.items] == tagged_ids[1:] + + restarted = await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery(tag_filter=TagFilter(tags=("paged",)), limit=1), + ) + changed_target = MemoryEntryTagTarget(artifact_id=first.artifact_id, entry_id=tagged_ids[1]) + changed = await contexts.records.get_tags("project", changed_target) + await contexts.records.replace_tags("project", changed_target, (), expected_etag=changed.etag) + with pytest.raises(InvalidCursorError, match="tag_state_changed"): + await service.query_directory( + first.artifact_id, + MemoryDirectoryQuery( + tag_filter=TagFilter(tags=("paged",)), + limit=1, + cursor=restarted.next_cursor, + ), + ) + + asyncio.run(scenario()) + + def test_sqlite_memory_backend_commits_authoritative_history_and_fts() -> None: async def scenario() -> None: async with open_builtin_contexts(BuiltinConfig(database=SQLiteConfig())) as contexts: From ca6e2c19d2bf9d7f5dad94c708659626907f11dc Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:43:20 +0800 Subject: [PATCH 04/14] feat(memory): gate directory queries on verified migration --- .../1656_bounded_memory_entry_pagination.md | 7 +- .../1656_bounded_memory_entry_pagination.md | 6 +- .../builtin/persistence/memory.py | 2 + .../persistence/memory_query_migration.py | 491 ++++++++++++++++++ .../builtin/persistence/tables.py | 28 + .../builtin/runtime/composition.py | 3 + tests/builtin/persistence/test_memory.py | 4 + .../test_memory_query_migration.py | 178 +++++++ 8 files changed, 714 insertions(+), 5 deletions(-) create mode 100644 src/powercontext/builtin/persistence/memory_query_migration.py create mode 100644 tests/builtin/persistence/test_memory_query_migration.py diff --git a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md index 91480608e1..7b4ccc579a 100644 --- a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md @@ -4,7 +4,7 @@ - Related work: [#1657](https://github.com/oceanbase/powercontext/issues/1657), [#1709](https://github.com/oceanbase/powercontext/pull/1709), and [#1718](https://github.com/oceanbase/powercontext/issues/1718) -- Status: core design accepted in principle; clarified contract specified; implementation not started +- Status: core design accepted in principle; implementation in progress # Summary @@ -237,5 +237,6 @@ SQLite scale measurements; OceanBase evidence when its service is available; gen format, lint, and type checks; then `make check` and `make unit-test`. Report unavailable services and skipped checks separately from passes. -Current status: this SPEC is checked against the issue discussion and current call paths. No feature code or feature -tests exist yet, and no implementation check is claimed as passing. +Current status: the revision-valid directory, tag-generation invalidation, bounded runtime query, and feature-scoped +SQLite migration/readiness gate are implemented with focused persistence and runtime tests. The public OpenAPI, +Server/client surface, operator CLI, scale report, OceanBase migration evidence, and full repository checks remain. diff --git a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md index 0e3161ad41..00cc1a080c 100644 --- a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md @@ -4,7 +4,7 @@ - 关联工作：[#1657](https://github.com/oceanbase/powercontext/issues/1657)、 [#1709](https://github.com/oceanbase/powercontext/pull/1709) 和 [#1718](https://github.com/oceanbase/powercontext/issues/1718) -- 状态：核心设计原则已达成共识；澄清后的合同已规定；实现尚未开始 +- 状态：核心设计原则已达成共识；实现进行中 # 摘要 @@ -210,4 +210,6 @@ review 前运行定向 OpenAPI、cursor、Memory persistence、tag、migration 可用时的 OceanBase 证据；generated-code、contract、docs、format、lint、type check；以及 `make check` 和 `make unit-test`。 不可用服务和 skipped check 必须与 pass 分开报告。 -当前状态：本 SPEC 已对照 issue 讨论和当前调用路径审查。功能代码和功能测试尚未实现，不声称任何实现检查已通过。 +当前状态：revision-valid directory、tag generation 失效、有界 runtime query 和 feature-scoped SQLite +migration/readiness gate 已实现，并通过定向 persistence/runtime 测试。公开 OpenAPI、Server/client +接口、运维 CLI、规模报告、OceanBase migration 证据和全仓检查仍待完成。 diff --git a/src/powercontext/builtin/persistence/memory.py b/src/powercontext/builtin/persistence/memory.py index 36a10cdde9..89a4f0a2a1 100644 --- a/src/powercontext/builtin/persistence/memory.py +++ b/src/powercontext/builtin/persistence/memory.py @@ -65,6 +65,7 @@ from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.errors import RepositoryNotFoundError from powercontext.builtin.persistence.memory_index import MemoryIndex, NoMemoryIndex +from powercontext.builtin.persistence.memory_query_migration import require_memory_query_index from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, MEMORY_ENTRY_DIRECTORY_TABLE, @@ -223,6 +224,7 @@ async def _select_directory_rows( after_entry_id: str, ) -> tuple[int, int | None, list[Mapping[Any, Any]]] | None: async with self._database.transaction() as connection: + await require_memory_query_index(connection) current_revision = await connection.scalar( select(ARTIFACT_HEADS_TABLE.c.revision).where( ARTIFACT_HEADS_TABLE.c.scope_id == self._scope_id, diff --git a/src/powercontext/builtin/persistence/memory_query_migration.py b/src/powercontext/builtin/persistence/memory_query_migration.py new file mode 100644 index 0000000000..6c548cdd70 --- /dev/null +++ b/src/powercontext/builtin/persistence/memory_query_migration.py @@ -0,0 +1,491 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explicit, resumable migration for bounded Memory directory queries. + +Normal startup creates the projection tables and marks only a data-empty +database ready. Existing Memory data stays available through the legacy APIs, +but the bounded directory query remains gated until an operator applies and +verifies this migration. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from pydantic import BaseModel +from sqlalchemy import and_, func, insert, or_, select, update +from sqlalchemy.dialects.mysql import insert as mysql_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.artifacts.memory import Memory, MemoryContent +from powercontext.builtin.persistence.codec import load_model, stored_bytes +from powercontext.builtin.persistence.errors import PersistenceError +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACTS_TABLE, + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, +) + +MEMORY_QUERY_INDEX_SCHEMA_VERSION = 1656 +_MAX_VERIFICATION_ISSUES = 32 +_VERIFICATION_PAGE_SIZE = 100 + + +class MemoryQueryIndexUnavailableError(PersistenceError): + """The bounded query projection has not completed verified migration.""" + + def __init__(self, code: str = "unavailable") -> None: + messages = { + "unavailable": "Memory query index is unavailable; complete offline plan/apply/verify first", + "migration-id": "Migration ID must contain 1 to 64 characters", + "batch-size": "Migration batch size must be a positive integer", + "marker": "Memory query migration marker is unavailable", + "version": "Unsupported Memory query index schema version", + "resume": "Resume the unfinished migration with its original ID", + "dialect": "Unsupported Memory query migration database dialect", + "conflict": "Existing directory row conflicts with authoritative Memory", + } + self.reason = messages[code] + super().__init__(self.reason) + + +class MemoryQueryMigrationPlan(BaseModel): + schema_version: int = MEMORY_QUERY_INDEX_SCHEMA_VERSION + phase: str + required: bool + memory_revision_count: int + directory_row_count: int + + +class MemoryQueryMigrationProgress(BaseModel): + phase: str + processed: int = 0 + complete: bool = False + + +class MemoryQueryMigrationVerification(BaseModel): + ready: bool + checked_revisions: int = 0 + issues: tuple[str, ...] = () + + +async def bootstrap_memory_query_index(connection: AsyncConnection) -> None: + """Mark a data-empty schema ready while leaving legacy Memory data gated.""" + + marker = await _marker(connection) + if marker is not None: + return + await _seed_marker(connection) + marker = await _marker(connection, for_update=True) + if marker is None or marker["phase"] == "complete": + return + memory_count = await connection.scalar( + select(func.count()).select_from(ARTIFACT_HEADS_TABLE).where(ARTIFACT_HEADS_TABLE.c.family == Memory.family) + ) + if int(memory_count or 0) == 0: + await connection.execute( + update(MEMORY_QUERY_INDEX_SCHEMA_TABLE) + .where(MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.singleton == 1) + .values(phase="complete", migration_id="fresh") + ) + + +async def require_memory_query_index(connection: AsyncConnection) -> None: + """Fail closed unless the exact query-index schema completed verification.""" + + marker = await _marker(connection) + if ( + marker is None + or int(marker["schema_version"]) != MEMORY_QUERY_INDEX_SCHEMA_VERSION + or marker["phase"] != "complete" + ): + raise MemoryQueryIndexUnavailableError + + +async def plan_memory_query_migration(connection: AsyncConnection) -> MemoryQueryMigrationPlan: + marker = await _marker(connection) + phase = "uninitialized" if marker is None else str(marker["phase"]) + revisions = await connection.scalar( + select(func.count()).select_from(ARTIFACTS_TABLE).where(ARTIFACTS_TABLE.c.family == Memory.family) + ) + directory = await connection.scalar(select(func.count()).select_from(MEMORY_ENTRY_DIRECTORY_TABLE)) + return MemoryQueryMigrationPlan( + phase=phase, + required=marker is None + or int(marker["schema_version"]) != MEMORY_QUERY_INDEX_SCHEMA_VERSION + or marker["phase"] != "complete", + memory_revision_count=int(revisions or 0), + directory_row_count=int(directory or 0), + ) + + +async def apply_memory_query_migration( # noqa: C901 - explicit durable maintenance phases + connection: AsyncConnection, + *, + migration_id: str = "rfc1656", + batch_size: int = 100, +) -> MemoryQueryMigrationProgress: + """Apply one bounded revision batch; the caller owns the transaction.""" + + if not migration_id or len(migration_id) > 64: + raise MemoryQueryIndexUnavailableError("migration-id") + if not isinstance(batch_size, int) or isinstance(batch_size, bool) or batch_size < 1: + raise MemoryQueryIndexUnavailableError("batch-size") + + marker = await _marker(connection, for_update=True) + if marker is None: + await bootstrap_memory_query_index(connection) + marker = await _marker(connection, for_update=True) + if marker is None: + raise MemoryQueryIndexUnavailableError("marker") + if int(marker["schema_version"]) != MEMORY_QUERY_INDEX_SCHEMA_VERSION: + raise MemoryQueryIndexUnavailableError("version") + if marker["phase"] == "complete": + return MemoryQueryMigrationProgress(phase="complete", complete=True) + if marker["phase"] == "bootstrap": + await connection.execute( + update(MEMORY_QUERY_INDEX_SCHEMA_TABLE) + .where(MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.singleton == 1) + .values(phase="backfill", migration_id=migration_id) + ) + marker = {**marker, "phase": "backfill", "migration_id": migration_id} + elif marker["migration_id"] != migration_id: + raise MemoryQueryIndexUnavailableError("resume") + + rows = ( + ( + await connection.execute( + _remaining_revisions(marker) + .order_by( + ARTIFACTS_TABLE.c.scope_id, + ARTIFACTS_TABLE.c.artifact_id, + ARTIFACTS_TABLE.c.revision, + ) + .limit(batch_size) + ) + ) + .mappings() + .all() + ) + if not rows: + await connection.execute( + update(MEMORY_QUERY_INDEX_SCHEMA_TABLE) + .where(MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.singleton == 1) + .values(phase="verify") + ) + return MemoryQueryMigrationProgress(phase="verify") + + for row in rows: + await _backfill_revision(connection, row) + last = rows[-1] + await connection.execute( + update(MEMORY_QUERY_INDEX_SCHEMA_TABLE) + .where(MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.singleton == 1) + .values( + checkpoint_scope_id=last["scope_id"], + checkpoint_artifact_id=last["artifact_id"], + checkpoint_revision=last["revision"], + ) + ) + return MemoryQueryMigrationProgress(phase="backfill", processed=len(rows)) + + +async def verify_memory_query_migration( # noqa: C901 - independent persisted invariants + connection: AsyncConnection, +) -> MemoryQueryMigrationVerification: + """Verify exact revision views and mark the feature ready only on success.""" + + marker = await _marker(connection, for_update=True) + if marker is None: + return MemoryQueryMigrationVerification(ready=False, issues=("missing migration marker",)) + issues: list[str] = [] + if int(marker["schema_version"]) != MEMORY_QUERY_INDEX_SCHEMA_VERSION: + issues.append("unsupported schema version") + if marker["phase"] not in {"verify", "complete"}: + issues.append("migration backfill is incomplete") + checked = 0 + if not issues: + after: tuple[str, str, int] | None = None + while len(issues) < _MAX_VERIFICATION_ISSUES: + rows = ( + ( + await connection.execute( + _revisions_after(after) + .order_by( + ARTIFACTS_TABLE.c.scope_id, + ARTIFACTS_TABLE.c.artifact_id, + ARTIFACTS_TABLE.c.revision, + ) + .limit(_VERIFICATION_PAGE_SIZE) + ) + ) + .mappings() + .all() + ) + if not rows: + break + for row in rows: + checked += 1 + content = _memory_content(row) + actual = await _directory_at_revision( + connection, + str(row["scope_id"]), + str(row["artifact_id"]), + int(row["revision"]), + ) + expected = {(item.entry_id, item.entry_version_id, item.state) for item in content.manifest.entries} + if actual != expected: + issues.append(f"directory mismatch at {row['scope_id']}/{row['artifact_id']}@{row['revision']}") + if len(issues) >= _MAX_VERIFICATION_ISSUES: + break + last = rows[-1] + after = (str(last["scope_id"]), str(last["artifact_id"]), int(last["revision"])) + missing_generations = await connection.scalar( + select(func.count()) + .select_from(ARTIFACT_HEADS_TABLE) + .where( + ARTIFACT_HEADS_TABLE.c.family == Memory.family, + ~select(MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id) + .where( + MEMORY_TAG_GENERATIONS_TABLE.c.scope_id == ARTIFACT_HEADS_TABLE.c.scope_id, + MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id == ARTIFACT_HEADS_TABLE.c.artifact_id, + ) + .exists(), + ) + ) + if missing_generations: + issues.append("Memory heads remain without tag generations") + if issues: + return MemoryQueryMigrationVerification(ready=False, checked_revisions=checked, issues=tuple(issues)) + + await connection.execute( + update(MEMORY_QUERY_INDEX_SCHEMA_TABLE) + .where(MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.singleton == 1) + .values(phase="complete") + ) + return MemoryQueryMigrationVerification(ready=True, checked_revisions=checked) + + +async def _seed_marker(connection: AsyncConnection) -> None: + values = { + "singleton": 1, + "schema_version": MEMORY_QUERY_INDEX_SCHEMA_VERSION, + "phase": "bootstrap", + "migration_id": "pending", + "checkpoint_scope_id": None, + "checkpoint_artifact_id": None, + "checkpoint_revision": None, + } + if connection.dialect.name == "sqlite": + statement = sqlite_insert(MEMORY_QUERY_INDEX_SCHEMA_TABLE).values(**values).on_conflict_do_nothing() + elif connection.dialect.name == "mysql": + statement = ( + mysql_insert(MEMORY_QUERY_INDEX_SCHEMA_TABLE) + .values(**values) + .on_duplicate_key_update(schema_version=MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.schema_version) + ) + else: + raise MemoryQueryIndexUnavailableError("dialect") + await connection.execute(statement) + + +def _remaining_revisions(marker: Mapping[str, Any]): + checkpoint = None + if ( + marker["checkpoint_scope_id"] is not None + and marker["checkpoint_artifact_id"] is not None + and marker["checkpoint_revision"] is not None + ): + checkpoint = ( + str(marker["checkpoint_scope_id"]), + str(marker["checkpoint_artifact_id"]), + int(marker["checkpoint_revision"]), + ) + return _revisions_after(checkpoint) + + +def _revisions_after(checkpoint: tuple[str, str, int] | None): + statement = select( + ARTIFACTS_TABLE.c.scope_id, + ARTIFACTS_TABLE.c.artifact_id, + ARTIFACTS_TABLE.c.revision, + ARTIFACTS_TABLE.c.content, + ).where(ARTIFACTS_TABLE.c.family == Memory.family) + if checkpoint is None: + return statement + scope_id, artifact_id, revision = checkpoint + return statement.where( + or_( + ARTIFACTS_TABLE.c.scope_id > scope_id, + and_( + ARTIFACTS_TABLE.c.scope_id == scope_id, + ARTIFACTS_TABLE.c.artifact_id > artifact_id, + ), + and_( + ARTIFACTS_TABLE.c.scope_id == scope_id, + ARTIFACTS_TABLE.c.artifact_id == artifact_id, + ARTIFACTS_TABLE.c.revision > revision, + ), + ) + ) + + +async def _backfill_revision(connection: AsyncConnection, row: Mapping[Any, Any]) -> None: + scope_id = str(row["scope_id"]) + artifact_id = str(row["artifact_id"]) + revision = int(row["revision"]) + content = _memory_content(row) + manifest = {item.entry_id: item for item in content.manifest.entries} + previous = ( + {} + if revision == 1 + else { + entry_id: (entry_version_id, state) + for entry_id, entry_version_id, state in await _directory_at_revision( + connection, + scope_id, + artifact_id, + revision - 1, + ) + } + ) + current = {entry_id: (item.entry_version_id, item.state) for entry_id, item in manifest.items()} + changed_ids = { + entry_id for entry_id in previous.keys() | current.keys() if previous.get(entry_id) != current.get(entry_id) + } + + for entry_id in sorted(changed_ids): + await connection.execute( + update(MEMORY_ENTRY_DIRECTORY_TABLE) + .where( + MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == scope_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == artifact_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id == entry_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision < revision, + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_to_revision.is_(None), + ) + .values(valid_to_revision=revision) + ) + item = manifest.get(entry_id) + if item is None: + continue + existing = ( + ( + await connection.execute( + select(MEMORY_ENTRY_DIRECTORY_TABLE).where( + MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == scope_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == artifact_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id == entry_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision == revision, + ) + ) + ) + .mappings() + .one_or_none() + ) + expected = { + "entry_version_id": item.entry_version_id, + "state": item.state, + } + if existing is None: + await connection.execute( + insert(MEMORY_ENTRY_DIRECTORY_TABLE).values( + scope_id=scope_id, + family=Memory.family, + memory_artifact_id=artifact_id, + entry_id=item.entry_id, + entry_version_id=item.entry_version_id, + state=item.state, + valid_from_revision=revision, + valid_to_revision=None, + ) + ) + elif any(existing[key] != value for key, value in expected.items()): + raise MemoryQueryIndexUnavailableError("conflict") + + generation = await connection.scalar( + select(MEMORY_TAG_GENERATIONS_TABLE.c.generation).where( + MEMORY_TAG_GENERATIONS_TABLE.c.scope_id == scope_id, + MEMORY_TAG_GENERATIONS_TABLE.c.memory_artifact_id == artifact_id, + ) + ) + if generation is None: + await connection.execute( + insert(MEMORY_TAG_GENERATIONS_TABLE).values( + scope_id=scope_id, + family=Memory.family, + memory_artifact_id=artifact_id, + generation=0, + ) + ) + + +def _memory_content(row: Mapping[Any, Any]) -> MemoryContent: + return load_model( + MemoryContent, + stored_bytes(row["content"], column="payload"), + kind="artifact", + name=Memory.family, + ) + + +async def _directory_at_revision( + connection: AsyncConnection, + scope_id: str, + artifact_id: str, + revision: int, +) -> set[tuple[str, str, str]]: + rows = await connection.execute( + select( + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_version_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.state, + ).where( + MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == scope_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == artifact_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision <= revision, + or_( + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_to_revision.is_(None), + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_to_revision > revision, + ), + ) + ) + return {(str(entry_id), str(entry_version_id), str(state)) for entry_id, entry_version_id, state in rows} + + +async def _marker(connection: AsyncConnection, *, for_update: bool = False) -> Mapping[str, Any] | None: + statement = select(MEMORY_QUERY_INDEX_SCHEMA_TABLE).where(MEMORY_QUERY_INDEX_SCHEMA_TABLE.c.singleton == 1) + if for_update: + statement = statement.with_for_update() + row = (await connection.execute(statement)).mappings().one_or_none() + return None if row is None else dict(row) + + +__all__ = [ + "MEMORY_QUERY_INDEX_SCHEMA_VERSION", + "MemoryQueryIndexUnavailableError", + "MemoryQueryMigrationPlan", + "MemoryQueryMigrationProgress", + "MemoryQueryMigrationVerification", + "apply_memory_query_migration", + "bootstrap_memory_query_index", + "plan_memory_query_migration", + "require_memory_query_index", + "verify_memory_query_migration", +] diff --git a/src/powercontext/builtin/persistence/tables.py b/src/powercontext/builtin/persistence/tables.py index 28f1d6045c..ba6f87c8e1 100644 --- a/src/powercontext/builtin/persistence/tables.py +++ b/src/powercontext/builtin/persistence/tables.py @@ -1085,11 +1085,39 @@ def _entry_text_type(): ) +MEMORY_QUERY_INDEX_SCHEMA_TABLE = Table( + "pc_memory_query_index_schema", + SHARED_METADATA, + Column("singleton", Integer, primary_key=True, autoincrement=False), + Column("schema_version", Integer, nullable=False), + Column("phase", identity_string(16), nullable=False), + Column("migration_id", identity_string(64), nullable=False), + Column("checkpoint_scope_id", identity_string(MAX_SCOPE_ID_LENGTH)), + Column("checkpoint_artifact_id", identity_string(MAX_ARTIFACT_ID_LENGTH)), + Column("checkpoint_revision", Integer), + CheckConstraint("singleton = 1", name="ck_pc_memory_query_schema_singleton"), + CheckConstraint( + "phase IN ('bootstrap', 'backfill', 'verify', 'complete')", + name="ck_pc_memory_query_schema_phase", + ), + CheckConstraint( + "checkpoint_revision IS NULL OR checkpoint_revision > 0", + name="ck_pc_memory_query_schema_checkpoint_revision", + ), + CheckConstraint( + "(checkpoint_scope_id IS NULL AND checkpoint_artifact_id IS NULL AND checkpoint_revision IS NULL) OR " + "(checkpoint_scope_id IS NOT NULL AND checkpoint_artifact_id IS NOT NULL AND checkpoint_revision IS NOT NULL)", + name="ck_pc_memory_query_schema_checkpoint_complete", + ), +) + + MEMORY_TABLES = ( MEMORY_ENTRY_VERSIONS_TABLE, MEMORY_ENTRY_DIRECTORY_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_TAG_GENERATIONS_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, ) TOPIC_MEMORY_TABLES = ( diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index ed98185ea7..07e0cdd83b 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -79,6 +79,7 @@ ) from powercontext.builtin.persistence.dream_schema import ensure_dream_schema from powercontext.builtin.persistence.memory_index import CompositeMemoryIndex, MemoryIndex +from powercontext.builtin.persistence.memory_query_migration import bootstrap_memory_query_index from powercontext.builtin.persistence.oceanbase.experience_index import OceanBaseExperienceFTSIndex from powercontext.builtin.persistence.oceanbase.memory_index import ( OceanBaseMemoryFTSIndex, @@ -846,6 +847,7 @@ async def open_builtin_contexts( async with profile.database.transaction() as connection: await bootstrap_processing_schema(connection, canonical_processing_manifest(config)) await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) + await bootstrap_memory_query_index(connection) await ensure_skill_distribution_schema(connection) await ensure_topic_memory_tag_schema(connection) await ensure_dream_schema(connection) @@ -908,6 +910,7 @@ async def open_builtin_contexts( async with profile.database.transaction() as connection: await bootstrap_processing_schema(connection, canonical_processing_manifest(config)) await assert_processing_schema_ready(connection, canonical_processing_manifest(config)) + await bootstrap_memory_query_index(connection) await ensure_skill_distribution_schema(connection) await ensure_topic_memory_tag_schema(connection) await ensure_dream_schema(connection) diff --git a/tests/builtin/persistence/test_memory.py b/tests/builtin/persistence/test_memory.py index 9c61485d06..8c872cbb96 100644 --- a/tests/builtin/persistence/test_memory.py +++ b/tests/builtin/persistence/test_memory.py @@ -27,6 +27,7 @@ MEMORY_ENTRY_DIRECTORY_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_ENTRY_VERSIONS_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, MEMORY_TAG_GENERATIONS_TABLE, ) from powercontext.builtin.records import InvalidCursorError @@ -61,6 +62,7 @@ def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() directory = str(CreateTable(MEMORY_ENTRY_DIRECTORY_TABLE).compile(dialect=dialect)) heads = str(CreateTable(MEMORY_ENTRY_HEADS_TABLE).compile(dialect=dialect)) generations = str(CreateTable(MEMORY_TAG_GENERATIONS_TABLE).compile(dialect=dialect)) + migration = str(CreateTable(MEMORY_QUERY_INDEX_SCHEMA_TABLE).compile(dialect=dialect)) assert "scope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL" in versions assert "text MEDIUMTEXT NOT NULL" in versions @@ -70,6 +72,7 @@ def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() assert "valid_to_revision INTEGER" in directory assert "searchable_text MEDIUMTEXT NOT NULL" in heads assert "generation BIGINT NOT NULL" in generations + assert "phase VARCHAR(16)" in migration budgets = [ _key_budget(constraint) @@ -78,6 +81,7 @@ def test_memory_schema_is_mysql_compilable_and_respects_key_and_payload_limits() MEMORY_ENTRY_DIRECTORY_TABLE, MEMORY_ENTRY_HEADS_TABLE, MEMORY_TAG_GENERATIONS_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, ) for constraint in table.constraints if isinstance(constraint, PrimaryKeyConstraint | UniqueConstraint | ForeignKeyConstraint) diff --git a/tests/builtin/persistence/test_memory_query_migration.py b/tests/builtin/persistence/test_memory_query_migration.py new file mode 100644 index 0000000000..ca5ec8ca1c --- /dev/null +++ b/tests/builtin/persistence/test_memory_query_migration.py @@ -0,0 +1,178 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio + +import pytest +from sqlalchemy import delete, select, update + +from powercontext.builtin.artifacts.memory import MemoryDirectoryQuery, MemoryEntryInput +from powercontext.builtin.persistence.memory_query_migration import ( + MemoryQueryIndexUnavailableError, + apply_memory_query_migration, + plan_memory_query_migration, + verify_memory_query_migration, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + BUILTIN_TABLES, + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, +) +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts + + +def test_existing_memory_requires_resumable_verified_directory_backfill(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'memory-query-migration.db'}") + runtime_config = BuiltinConfig(database=config) + + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + first = await service.remember( + memory=None, + entries=( + MemoryEntryInput(kind="fact", text="Keep this entry."), + MemoryEntryInput(kind="decision", text="Revise this entry before upgrading."), + ), + mode="append", + ) + assert first is not None + entries = await service.entries(first) + changing = next(entry for entry in entries if entry.kind == "decision") + second = await service.remember( + memory=first, + entries=( + MemoryEntryInput( + entry=changing, + kind=changing.kind, + text="This is the pre-upgrade revision.", + ), + ), + mode="append", + ) + assert second is not None + + # Simulate a database created before the directory projection existed. + async with ( + SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile, + profile.database.transaction() as connection, + ): + await connection.execute(delete(MEMORY_ENTRY_DIRECTORY_TABLE)) + await connection.execute(delete(MEMORY_TAG_GENERATIONS_TABLE)) + await connection.execute(delete(MEMORY_QUERY_INDEX_SCHEMA_TABLE)) + + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + assert len(await service.entries(second)) == 2 + with pytest.raises(MemoryQueryIndexUnavailableError): + await service.query_directory(second.artifact_id, MemoryDirectoryQuery()) + + # Legacy writes remain available and maintain their delta even + # while historical revisions still require backfill. + stable = next(entry for entry in await service.entries(second) if entry.kind == "fact") + third = await service.remember( + memory=second, + entries=( + MemoryEntryInput( + entry=stable, + kind=stable.kind, + text="This post-upgrade write already owns its directory delta.", + ), + ), + mode="append", + ) + assert third is not None + assert len(await service.entries(third)) == 2 + + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile: + async with profile.database.transaction() as connection: + plan = await plan_memory_query_migration(connection) + assert plan.required + assert plan.phase == "bootstrap" + assert plan.memory_revision_count == 3 + assert plan.directory_row_count == 1 + + async with profile.database.transaction() as connection: + first_batch = await apply_memory_query_migration( + connection, + migration_id="test-1656", + batch_size=1, + ) + assert first_batch.phase == "backfill" + assert first_batch.processed == 1 + + async with profile.database.transaction() as connection: + with pytest.raises(MemoryQueryIndexUnavailableError, match="original ID"): + await apply_memory_query_migration(connection, migration_id="different", batch_size=1) + + while True: + async with profile.database.transaction() as connection: + progress = await apply_memory_query_migration( + connection, + migration_id="test-1656", + batch_size=1, + ) + if progress.phase == "verify": + break + + async with profile.database.transaction() as connection: + active_row = ( + ( + await connection.execute( + select(MEMORY_ENTRY_DIRECTORY_TABLE).where(MEMORY_ENTRY_DIRECTORY_TABLE.c.state == "active") + ) + ) + .mappings() + .first() + ) + assert active_row is not None + row_key = { + "scope_id": active_row["scope_id"], + "memory_artifact_id": active_row["memory_artifact_id"], + "entry_id": active_row["entry_id"], + "valid_from_revision": active_row["valid_from_revision"], + } + predicate = tuple( + getattr(MEMORY_ENTRY_DIRECTORY_TABLE.c, name) == value for name, value in row_key.items() + ) + await connection.execute( + update(MEMORY_ENTRY_DIRECTORY_TABLE).where(*predicate).values(state="inactive") + ) + failed = await verify_memory_query_migration(connection) + assert not failed.ready + assert failed.issues and failed.issues[0].startswith("directory mismatch") + await connection.execute(update(MEMORY_ENTRY_DIRECTORY_TABLE).where(*predicate).values(state="active")) + verified = await verify_memory_query_migration(connection) + assert verified.ready + assert verified.checked_revisions == 3 + + async with profile.database.transaction() as connection: + plan = await plan_memory_query_migration(connection) + assert not plan.required + assert plan.phase == "complete" + assert await connection.scalar(select(MEMORY_TAG_GENERATIONS_TABLE.c.generation)) == 0 + + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + page = await service.query_directory(third.artifact_id, MemoryDirectoryQuery(limit=10)) + assert page.memory_ref == third.as_ref() + assert {item.citation.entry_id for item in page.items} == { + entry.entry_id for entry in await service.entries(third) + } + + asyncio.run(scenario()) From f64ce411e45a7b7a308bf27ce2a6c6dc947274ae Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:50:37 +0800 Subject: [PATCH 05/14] feat(server): add memory query migration command --- docs/en/docs/operate/index.md | 1 + .../en/docs/operate/memory-query-migration.md | 53 ++++++++++++ docs/en/docs/operate/meta.json | 1 + docs/zh/docs/operate/index.md | 1 + .../zh/docs/operate/memory-query-migration.md | 42 ++++++++++ docs/zh/docs/operate/meta.json | 1 + .../persistence/memory_query_migration.py | 62 +++++++++++--- src/powercontext/server/cli.py | 71 ++++++++++++++++ tests/test_memory_query_migration_cli.py | 80 +++++++++++++++++++ 9 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 docs/en/docs/operate/memory-query-migration.md create mode 100644 docs/zh/docs/operate/memory-query-migration.md create mode 100644 tests/test_memory_query_migration_cli.py diff --git a/docs/en/docs/operate/index.md b/docs/en/docs/operate/index.md index b05f22c974..f05c6a6c0f 100644 --- a/docs/en/docs/operate/index.md +++ b/docs/en/docs/operate/index.md @@ -11,6 +11,7 @@ Start with the operational task you need: | --- | --- | | Keep a personal Server running, deploy a container, or enable authentication | [Deploy the Server](deploy-server.md) | | Upgrade or switch background supervision mode | [Migrate Artifact processing state](artifact-processing-migration.md) | +| Enable bounded Memory-entry queries after upgrading an existing database | [Migrate the Memory query index](memory-query-migration.md) | | Inspect logs, metrics, and traces | [Observability](observability.md) | | Send traces to an analysis service | [Phoenix](trace-with-phoenix.md), [Langfuse](trace-with-langfuse.md) | | Diagnose failures and restore service or data | [Troubleshoot](troubleshoot.md) | diff --git a/docs/en/docs/operate/memory-query-migration.md b/docs/en/docs/operate/memory-query-migration.md new file mode 100644 index 0000000000..e0a9ece3c6 --- /dev/null +++ b/docs/en/docs/operate/memory-query-migration.md @@ -0,0 +1,53 @@ +--- +title: Migrate the Memory query index +--- + +The bounded Memory-entry query uses a rebuildable directory projection. A new +database is marked ready automatically. After upgrading a database that already +contains Memory revisions, the Server keeps legacy list, exact-detail, search, +and write operations available, but the new query returns +`memory_query_index_unavailable` until this migration is verified. + +## Plan and apply + +Use the same deployment environment file as the Server: + +```shell +powercontext server memory-query-migrate --action plan --env-file .env +``` + +The plan is read-only and reports missing projection tables, authoritative +Memory revision count, existing directory rows, and the durable phase. + +Back up the database. Stop every old Server and Memory writer, disable their +automatic restart, and keep them stopped for the maintenance window. The +command cannot establish those external conditions. Then run: + +```shell +powercontext server memory-query-migrate --action apply --env-file .env \ + --migration-id rfc1656 --batch-size 100 --maintenance-confirmed +powercontext server memory-query-migrate --action verify --env-file .env +``` + +`apply` creates any missing feature tables one at a time and commits at most the +selected number of Memory revisions per backfill transaction. It verifies the +result before reporting success. If it stops, repeat it with the same migration +ID. Do not clear the marker or directory rows; a different ID is rejected while +an earlier migration is incomplete. + +Only restart writers and Server replicas after the command reports +`ready: true`. The separately runnable `verify` action rechecks every +authoritative revision and tag-generation row and is safe to repeat. + +## What the migration changes + +The migration reads immutable Memory manifests and rebuilds revision-valid +compact directory rows plus one tag generation per Memory Artifact. It never +rewrites Memory Artifacts, entry versions, tags, or entry bodies. The directory +remains derived state and is not used as an authority until full verification +succeeds. + +Memory writes from the new version maintain their own directory deltas, but a +rolling deployment with old and new writers during this offline migration is +unsupported. SQLite migration requires a persistent database; in-memory SQLite +has no upgrade state to preserve. diff --git a/docs/en/docs/operate/meta.json b/docs/en/docs/operate/meta.json index 5e04efad00..0d20caacb6 100644 --- a/docs/en/docs/operate/meta.json +++ b/docs/en/docs/operate/meta.json @@ -5,6 +5,7 @@ "deploy-server", "connect-remote-server", "artifact-processing-migration", + "memory-query-migration", "observability", "trace-with-phoenix", "trace-with-langfuse", diff --git a/docs/zh/docs/operate/index.md b/docs/zh/docs/operate/index.md index ed6a6b4af0..5e3769e649 100644 --- a/docs/zh/docs/operate/index.md +++ b/docs/zh/docs/operate/index.md @@ -11,6 +11,7 @@ description: 持续运行 Server，查找部署、监控与恢复步骤。 | --- | --- | | 持续运行个人 Server、部署容器或启用认证 | [部署 Server](deploy-server.md) | | 升级或切换后台监督模式 | [迁移 Artifact 处理状态](artifact-processing-migration.md) | +| 升级现有数据库后启用有界 Memory entry 查询 | [迁移 Memory 查询索引](memory-query-migration.md) | | 查看日志、指标和 Trace | [可观测性](observability.md) | | 将 Trace 发送到分析服务 | [Phoenix](trace-with-phoenix.md)、[Langfuse](trace-with-langfuse.md) | | 诊断失败并恢复服务或数据 | [诊断与恢复](troubleshoot.md) | diff --git a/docs/zh/docs/operate/memory-query-migration.md b/docs/zh/docs/operate/memory-query-migration.md new file mode 100644 index 0000000000..4da89c0c52 --- /dev/null +++ b/docs/zh/docs/operate/memory-query-migration.md @@ -0,0 +1,42 @@ +--- +title: 迁移 Memory 查询索引 +--- + +有界 Memory entry 查询使用可重建的 directory 投影。新数据库会自动标记为 ready。 +已有 Memory revision 的数据库升级后，Server 仍可使用旧的全量列表、精确详情、搜索和写入； +但在本迁移验证成功前，新查询会返回 `memory_query_index_unavailable`。 + +## 查看计划并执行 + +使用与 Server 相同的部署环境文件： + +```shell +powercontext server memory-query-migrate --action plan --env-file .env +``` + +plan 只读，会报告缺失的投影表、权威 Memory revision 数、已有 directory 行数和持久化阶段。 + +先备份数据库，停止全部旧 Server 和 Memory writer，关闭它们的自动重启，并在维护窗口内保持停止。 +命令无法代替操作者建立这些外部条件。随后执行： + +```shell +powercontext server memory-query-migrate --action apply --env-file .env \ + --migration-id rfc1656 --batch-size 100 --maintenance-confirmed +powercontext server memory-query-migrate --action verify --env-file .env +``` + +`apply` 会逐张创建缺失的功能表，每个 backfill 事务最多提交指定数量的 Memory revision， +并在报告成功前验证结果。中断后使用相同 migration ID 重复执行即可。不要清除 marker 或 +directory 行；前一次迁移未完成时，更换 ID 会被拒绝。 + +只有命令返回 `ready: true` 后才能重启 writer 和 Server 副本。独立的 `verify` action +会重新检查所有权威 revision 和 tag generation 行，可安全重复执行。 + +## 迁移会改变什么 + +迁移读取不可变 Memory manifest，重建 revision-valid 紧凑 directory 行，并为每个 Memory Artifact +建立一个 tag generation。它不会改写 Memory Artifact、entry version、tag 或 entry body。 +directory 仍是派生状态，完整验证成功前不会成为查询依据。 + +新版本的 Memory 写入会维护自身 directory delta，但不支持在本离线迁移期间滚动混用新旧 writer。 +SQLite 迁移需要持久数据库；内存 SQLite 没有需要保留的升级状态。 diff --git a/docs/zh/docs/operate/meta.json b/docs/zh/docs/operate/meta.json index f8a7b06d97..493f66d664 100644 --- a/docs/zh/docs/operate/meta.json +++ b/docs/zh/docs/operate/meta.json @@ -5,6 +5,7 @@ "deploy-server", "connect-remote-server", "artifact-processing-migration", + "memory-query-migration", "observability", "trace-with-phoenix", "trace-with-langfuse", diff --git a/src/powercontext/builtin/persistence/memory_query_migration.py b/src/powercontext/builtin/persistence/memory_query_migration.py index 6c548cdd70..696c4c9571 100644 --- a/src/powercontext/builtin/persistence/memory_query_migration.py +++ b/src/powercontext/builtin/persistence/memory_query_migration.py @@ -26,7 +26,7 @@ from typing import Any from pydantic import BaseModel -from sqlalchemy import and_, func, insert, or_, select, update +from sqlalchemy import and_, func, insert, inspect, or_, select, update from sqlalchemy.dialects.mysql import insert as mysql_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import AsyncConnection @@ -45,6 +45,11 @@ MEMORY_QUERY_INDEX_SCHEMA_VERSION = 1656 _MAX_VERIFICATION_ISSUES = 32 _VERIFICATION_PAGE_SIZE = 100 +_MIGRATION_TABLES = ( + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, +) class MemoryQueryIndexUnavailableError(PersistenceError): @@ -69,6 +74,7 @@ class MemoryQueryMigrationPlan(BaseModel): schema_version: int = MEMORY_QUERY_INDEX_SCHEMA_VERSION phase: str required: bool + missing_tables: tuple[str, ...] = () memory_revision_count: int directory_row_count: int @@ -119,19 +125,20 @@ async def require_memory_query_index(connection: AsyncConnection) -> None: async def plan_memory_query_migration(connection: AsyncConnection) -> MemoryQueryMigrationPlan: - marker = await _marker(connection) + missing_tables = await _missing_migration_tables(connection) + marker = None if MEMORY_QUERY_INDEX_SCHEMA_TABLE.name in missing_tables else await _marker(connection) phase = "uninitialized" if marker is None else str(marker["phase"]) - revisions = await connection.scalar( - select(func.count()).select_from(ARTIFACTS_TABLE).where(ARTIFACTS_TABLE.c.family == Memory.family) - ) - directory = await connection.scalar(select(func.count()).select_from(MEMORY_ENTRY_DIRECTORY_TABLE)) + revisions = await _count_if_present(connection, ARTIFACTS_TABLE, family=Memory.family) + directory = await _count_if_present(connection, MEMORY_ENTRY_DIRECTORY_TABLE) return MemoryQueryMigrationPlan( phase=phase, - required=marker is None + required=bool(missing_tables) + or marker is None or int(marker["schema_version"]) != MEMORY_QUERY_INDEX_SCHEMA_VERSION or marker["phase"] != "complete", - memory_revision_count=int(revisions or 0), - directory_row_count=int(directory or 0), + missing_tables=missing_tables, + memory_revision_count=revisions, + directory_row_count=directory, ) @@ -148,6 +155,11 @@ async def apply_memory_query_migration( # noqa: C901 - explicit durable mainten if not isinstance(batch_size, int) or isinstance(batch_size, bool) or batch_size < 1: raise MemoryQueryIndexUnavailableError("batch-size") + for table in _MIGRATION_TABLES: + if not await _has_table(connection, table.name): + await connection.run_sync(lambda sync, owned=table: owned.create(sync, checkfirst=True)) + return MemoryQueryMigrationProgress(phase="schema") + marker = await _marker(connection, for_update=True) if marker is None: await bootstrap_memory_query_index(connection) @@ -211,6 +223,12 @@ async def verify_memory_query_migration( # noqa: C901 - independent persisted i ) -> MemoryQueryMigrationVerification: """Verify exact revision views and mark the feature ready only on success.""" + missing_tables = await _missing_migration_tables(connection) + if missing_tables: + return MemoryQueryMigrationVerification( + ready=False, + issues=tuple(f"missing {name}" for name in missing_tables), + ) marker = await _marker(connection, for_update=True) if marker is None: return MemoryQueryMigrationVerification(ready=False, issues=("missing migration marker",)) @@ -477,6 +495,32 @@ async def _marker(connection: AsyncConnection, *, for_update: bool = False) -> M return None if row is None else dict(row) +async def _has_table(connection: AsyncConnection, name: str) -> bool: + return bool(await connection.run_sync(lambda sync: inspect(sync).has_table(name))) + + +async def _missing_migration_tables(connection: AsyncConnection) -> tuple[str, ...]: + missing: list[str] = [] + for table in _MIGRATION_TABLES: + if not await _has_table(connection, table.name): + missing.append(table.name) + return tuple(missing) + + +async def _count_if_present( + connection: AsyncConnection, + table: Any, + *, + family: str | None = None, +) -> int: + if not await _has_table(connection, table.name): + return 0 + statement = select(func.count()).select_from(table) + if family is not None: + statement = statement.where(table.c.family == family) + return int(await connection.scalar(statement) or 0) + + __all__ = [ "MEMORY_QUERY_INDEX_SCHEMA_VERSION", "MemoryQueryIndexUnavailableError", diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 36a850cf36..6e08237c57 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -25,6 +25,11 @@ import typer from pydantic import ValidationError +from powercontext.builtin.persistence.memory_query_migration import ( + apply_memory_query_migration, + plan_memory_query_migration, + verify_memory_query_migration, +) from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.processing_migration import ( apply_processing_migration, @@ -161,6 +166,72 @@ async def _processing_maintenance( return verification.ready +@app.command("memory-query-migrate") +def memory_query_migrate( + action: Annotated[Literal["plan", "apply", "verify"], typer.Option(help="Offline migration action.")] = "plan", + env_file: Annotated[Path | None, typer.Option(help="Load deployment settings from this environment file.")] = None, + maintenance_confirmed: Annotated[ + bool, + typer.Option(help="Confirm all old hosts and Memory writers are stopped."), + ] = False, + migration_id: Annotated[str, typer.Option(help="Stable resume ID for this migration.")] = "rfc1656", + batch_size: Annotated[ + int, + typer.Option(min=1, max=10000, help="Maximum Memory revisions committed per migration step."), + ] = 100, +) -> None: + """Plan, apply or verify the resumable Memory query-index migration.""" + + if action == "apply" and not maintenance_confirmed: + raise typer.BadParameter("apply requires --maintenance-confirmed after stopping old hosts and writes") # noqa: TRY003 + with server_settings_context(env_file=env_file) as settings: + ready = asyncio.run( + _memory_query_maintenance(settings, action, migration_id=migration_id, batch_size=batch_size) + ) + if not ready: + raise typer.Exit(code=1) + + +async def _memory_query_maintenance( + settings: ServerSettings, + action: Literal["plan", "apply", "verify"], + *, + migration_id: str, + batch_size: int, +) -> bool: + database = settings.database + if isinstance(database, SQLiteConfig): + if database.is_in_memory: + raise typer.BadParameter("offline migration requires a persistent database") # noqa: TRY003 + opened = SQLiteProfile.open(database, tables=()) + elif isinstance(database, OceanBaseConfig): + opened = OceanBaseProfile.open(database, tables=()) + elif isinstance(database, SeekDBConfig): + opened = SeekDBProfile.open(database, tables=()) + else: + raise typer.BadParameter("unsupported migration database") # noqa: TRY003 + async with opened as profile: + if action == "plan": + async with profile.database.transaction() as connection: + plan = await plan_memory_query_migration(connection) + typer.echo(plan.model_dump_json()) + return True + if action == "apply": + while True: + async with profile.database.transaction() as connection: + progress = await apply_memory_query_migration( + connection, + migration_id=migration_id, + batch_size=batch_size, + ) + if progress.complete or progress.phase == "verify": + break + async with profile.database.transaction() as connection: + verification = await verify_memory_query_migration(connection) + typer.echo(verification.model_dump_json()) + return verification.ready + + @app.command() def run( host: Annotated[str | None, typer.Option(help="Address to bind.")] = None, diff --git a/tests/test_memory_query_migration_cli.py b/tests/test_memory_query_migration_cli.py new file mode 100644 index 0000000000..fee51f1b3d --- /dev/null +++ b/tests/test_memory_query_migration_cli.py @@ -0,0 +1,80 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline Memory query maintenance exposes explicit resumable state.""" + +import asyncio +import json +import os + +from click import unstyle +from typer.testing import CliRunner + +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + BUILTIN_TABLES, + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, +) +from powercontext.cli.app import create_cli +from powercontext.server.cli import app + + +def test_memory_query_plan_apply_verify_creates_schema_and_requires_maintenance(tmp_path, monkeypatch): + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'memory-query-maintenance.db'}") + migration_tables = { + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, + } + + async def setup(): + async with SQLiteProfile.open( + config, + tables=tuple(table for table in BUILTIN_TABLES if table not in migration_tables), + ): + pass + + asyncio.run(setup()) + for key in tuple(os.environ): + if key.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(key) + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_KIND", "sqlite") + monkeypatch.setenv("POWERCONTEXT_SERVER_DATABASE_URL", config.url) + cli = create_cli([app]) + runner = CliRunner() + command = ["server", "memory-query-migrate"] + + plan = runner.invoke(cli, [*command, "--action", "plan"]) + assert plan.exit_code == 0, plan.output + plan_value = json.loads(plan.output) + assert plan_value["required"] is True + assert set(plan_value["missing_tables"]) == {table.name for table in migration_tables} + + denied = runner.invoke(cli, [*command, "--action", "apply"]) + assert denied.exit_code == 2 + assert "maintenance-confirmed" in unstyle(denied.output) + + applied = runner.invoke(cli, [*command, "--action", "apply", "--maintenance-confirmed", "--batch-size", "1"]) + assert applied.exit_code == 0, applied.output + assert json.loads(applied.output)["ready"] is True + + verified = runner.invoke(cli, [*command, "--action", "verify"]) + assert verified.exit_code == 0, verified.output + assert json.loads(verified.output)["ready"] is True + + replay = runner.invoke(cli, [*command, "--action", "apply", "--maintenance-confirmed"]) + assert replay.exit_code == 0, replay.output + assert config.url not in plan.output + applied.output + verified.output + replay.output From 6189d6942b05110a46bdfbcc2cfc6f371b8d44a9 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:03:11 +0800 Subject: [PATCH 06/14] feat(memory): expose bounded directory query --- .../1656_bounded_memory_entry_pagination.md | 7 +- .../1656_bounded_memory_entry_pagination.md | 6 +- .../powercontext/src/operations.generated.ts | 1 + .../powercontext/src/operations.generated.ts | 1 + .../powercontext/src/operations.generated.ts | 1 + openapi/powercontext.yaml | 111 ++++++++++++++++++ src/powercontext/client/client.py | 8 ++ src/powercontext/http/__init__.py | 6 + src/powercontext/http/_generated/models.py | 32 +++++ .../http/_generated/operations.py | 31 +++++ src/powercontext/http/_generated/schema.py | 95 +++++++++++++++ src/powercontext/server/app.py | 44 +++++++ src/powercontext/server/mapping.py | 19 +++ src/powercontext/server/mcp.py | 5 +- tests/e2e/test_runtime_server.py | 49 +++++++- tests/test_access_mcp.py | 26 +++- tests/test_api_contract.py | 25 ++++ tests/test_mcp.py | 1 + 18 files changed, 459 insertions(+), 9 deletions(-) diff --git a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md index 7b4ccc579a..c67c80301b 100644 --- a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md @@ -237,6 +237,7 @@ SQLite scale measurements; OceanBase evidence when its service is available; gen format, lint, and type checks; then `make check` and `make unit-test`. Report unavailable services and skipped checks separately from passes. -Current status: the revision-valid directory, tag-generation invalidation, bounded runtime query, and feature-scoped -SQLite migration/readiness gate are implemented with focused persistence and runtime tests. The public OpenAPI, -Server/client surface, operator CLI, scale report, OceanBase migration evidence, and full repository checks remain. +Current status: the revision-valid directory, tag-generation invalidation, bounded runtime query, feature-scoped +migration/readiness gate, operator CLI, and public OpenAPI/Server/SDK/MCP surface are implemented with focused +persistence, contract, access, and runtime tests. The scale report, OceanBase migration evidence, and full repository +checks remain. diff --git a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md index 00cc1a080c..a96762eeb5 100644 --- a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md @@ -210,6 +210,6 @@ review 前运行定向 OpenAPI、cursor、Memory persistence、tag、migration 可用时的 OceanBase 证据；generated-code、contract、docs、format、lint、type check；以及 `make check` 和 `make unit-test`。 不可用服务和 skipped check 必须与 pass 分开报告。 -当前状态：revision-valid directory、tag generation 失效、有界 runtime query 和 feature-scoped SQLite -migration/readiness gate 已实现，并通过定向 persistence/runtime 测试。公开 OpenAPI、Server/client -接口、运维 CLI、规模报告、OceanBase migration 证据和全仓检查仍待完成。 +当前状态：revision-valid directory、tag generation 失效、有界 runtime query、feature-scoped +migration/readiness gate、运维 CLI 和公开 OpenAPI/Server/SDK/MCP 接口已实现，并通过定向 +persistence、contract、access 和 runtime 测试。规模报告、OceanBase migration 证据和全仓检查仍待完成。 diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 32e2223d57..5a66829e54 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -58,6 +58,7 @@ export const OPERATIONS = { remember_memory: { method: 'POST', path: '/v1/memory/remember', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_memory: { method: 'POST', path: '/v1/memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_memory_entries: { method: 'POST', path: '/v1/memory/entries/list', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + query_memory_entries: { method: 'POST', path: '/v1/memory/entries/query', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_memory_entry: { method: 'POST', path: '/v1/memory/entries/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, revise_memory_entry: { method: 'POST', path: '/v1/memory/entries/revise', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, retire_memory_entry: { method: 'POST', path: '/v1/memory/entries/retire', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 32e2223d57..5a66829e54 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -58,6 +58,7 @@ export const OPERATIONS = { remember_memory: { method: 'POST', path: '/v1/memory/remember', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_memory: { method: 'POST', path: '/v1/memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_memory_entries: { method: 'POST', path: '/v1/memory/entries/list', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + query_memory_entries: { method: 'POST', path: '/v1/memory/entries/query', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_memory_entry: { method: 'POST', path: '/v1/memory/entries/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, revise_memory_entry: { method: 'POST', path: '/v1/memory/entries/revise', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, retire_memory_entry: { method: 'POST', path: '/v1/memory/entries/retire', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 32e2223d57..5a66829e54 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -58,6 +58,7 @@ export const OPERATIONS = { remember_memory: { method: 'POST', path: '/v1/memory/remember', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_memory: { method: 'POST', path: '/v1/memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_memory_entries: { method: 'POST', path: '/v1/memory/entries/list', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, + query_memory_entries: { method: 'POST', path: '/v1/memory/entries/query', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, get_memory_entry: { method: 'POST', path: '/v1/memory/entries/get', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, revise_memory_entry: { method: 'POST', path: '/v1/memory/entries/revise', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, retire_memory_entry: { method: 'POST', path: '/v1/memory/entries/retire', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index 1461bf30ef..12f3d74116 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -1595,6 +1595,50 @@ paths: $ref: "#/components/responses/Unavailable" "500": $ref: "#/components/responses/InternalError" + /v1/memory/entries/query: + post: + tags: [memory] + summary: Query a bounded page of Memory entry identities + description: >- + Traverse compact Memory entry identities in entry_id order without loading entry bodies. Page one pins the + current Memory Revision and later pages retain that revision. Use this operation for a bounded inventory or + audit; use exact entry detail for text and evidence. Tag changes invalidate a filtered continuation and require + restarting from page one. An empty page with a null memory_ref means the Scope has no Memory. + operationId: query_memory_entries + x-powercontext-access: {action: scope.read, resource: {type: scope, scope-id-from: scope_id}} + x-powercontext-scope-mode: current + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/QueryMemoryEntriesRequest" + responses: + "200": + description: One bounded page pinned to an exact Memory Revision. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/QueryMemoryEntriesResponse" + "400": + $ref: "#/components/responses/BadRequest" + "410": + $ref: "#/components/responses/CursorExpired" + "413": + $ref: "#/components/responses/MemoryDirectoryItemTooLarge" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "422": + $ref: "#/components/responses/InvalidRequest" + "503": + $ref: "#/components/responses/Unavailable" + "500": + $ref: "#/components/responses/InternalError" /v1/memory/entries/get: post: tags: [memory] @@ -4413,6 +4457,15 @@ components: application/json: schema: $ref: "#/components/schemas/ErrorResponse" + MemoryDirectoryItemTooLarge: + description: One compact Memory directory item exceeds the public page byte budget. + headers: + X-PowerContext-Request-ID: + $ref: "#/components/headers/RequestId" + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" NotFound: description: The requested durable value was not found or is not observable. headers: @@ -8333,6 +8386,50 @@ components: type: array items: $ref: "#/components/schemas/MemoryEntry" + QueryMemoryEntriesRequest: + type: object + additionalProperties: false + required: [scope_id] + properties: + scope_id: + type: string + minLength: 1 + maxLength: 256 + pattern: '.*\S.*' + include_inactive: + type: boolean + default: false + description: Include inactive entries from the pinned Memory Revision for explicit audit. + tag_filter: + $ref: '#/components/schemas/TagFilter' + limit: + type: integer + minimum: 1 + maximum: 100 + default: 50 + cursor: + type: string + minLength: 1 + maxLength: 4096 + nullable: true + QueryMemoryEntriesResponse: + type: object + additionalProperties: false + required: [memory_ref, items, next_cursor] + properties: + memory_ref: + allOf: + - $ref: "#/components/schemas/ArtifactReference" + nullable: true + items: + type: array + maxItems: 100 + items: + $ref: "#/components/schemas/MemoryDirectoryItem" + next_cursor: + type: string + maxLength: 4096 + nullable: true ListArtifactCandidatesRequest: type: object additionalProperties: false @@ -8403,6 +8500,20 @@ components: type: array items: $ref: "#/components/schemas/ArtifactReference" + MemoryDirectoryItem: + type: object + additionalProperties: false + required: [citation, version, kind, state] + properties: + citation: + $ref: "#/components/schemas/MemoryCitation" + version: + type: integer + minimum: 1 + kind: + type: string + state: + $ref: "#/components/schemas/MemoryEntryState" MemoryMutationResponse: type: object additionalProperties: false diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index f450861bc2..5409b93d5a 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -141,6 +141,8 @@ PublishArtifactRequest, PublishRemoteSkillRequest, PutProfilePolicyRequest, + QueryMemoryEntriesRequest, + QueryMemoryEntriesResponse, ReadinessResponse, ReconcileRemoteSkillsRequest, ReconcileRemoteSkillsResponse, @@ -278,6 +280,7 @@ PUT_PROFILE_POLICY, QUERY_ARTIFACT_TAGS, QUERY_CODE, + QUERY_MEMORY_ENTRIES, RECONCILE_REMOTE_SKILLS, RECORD_REMOTE_SKILL_RECEIPT, RECORD_SKILL_USAGE, @@ -954,6 +957,11 @@ async def list_memory_entries(self, request: ListMemoryEntriesRequest) -> ListMe return await self._request(LIST_MEMORY_ENTRIES, request) + async def query_memory_entries(self, request: QueryMemoryEntriesRequest) -> QueryMemoryEntriesResponse: + """Query compact entry identities in a bounded revision-pinned page.""" + + return await self._request(QUERY_MEMORY_ENTRIES, request) + async def get_memory_entry(self, request: GetMemoryEntryRequest) -> MemoryEntry: """Read one exact Memory entry version.""" diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index 1a41691998..66d8acc23e 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -227,6 +227,7 @@ LiveStateCheckStatus, ManagedSkillLibraryEntry, MemoryCitation, + MemoryDirectoryItem, MemoryEntry, MemoryEntryAccessSelector, MemoryEntryInventoryStatistics, @@ -271,6 +272,8 @@ PublishRemoteSkillRequest, PutProfilePolicyRequest, QueryArtifactTagsRequest, + QueryMemoryEntriesRequest, + QueryMemoryEntriesResponse, ReadinessResponse, ReadinessStatus, RecallTokenDay, @@ -621,6 +624,7 @@ "LiveStateCheckStatus", "ManagedSkillLibraryEntry", "MemoryCitation", + "MemoryDirectoryItem", "MemoryEntry", "MemoryEntryAccessSelector", "MemoryEntryInventoryStatistics", @@ -665,6 +669,8 @@ "PublishRemoteSkillRequest", "PutProfilePolicyRequest", "QueryArtifactTagsRequest", + "QueryMemoryEntriesRequest", + "QueryMemoryEntriesResponse", "ReadinessResponse", "ReadinessStatus", "RecallTokenDay", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 5ac8ed0246..0d695ac299 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -3242,6 +3242,19 @@ class ListMemoryEntriesRequest(BaseModel): ] = False +class QueryMemoryEntriesRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + include_inactive: Annotated[ + StrictBool, Field(description="Include inactive entries from the pinned Memory Revision for explicit audit.") + ] = False + tag_filter: TagFilter | None = None + limit: Annotated[StrictInt, Field(ge=1, le=100)] = 50 + cursor: Annotated[StrictStr | None, Field(max_length=4096, min_length=1)] = None + + class ListArtifactCandidatesRequest(BaseModel): model_config = ConfigDict( extra="forbid", @@ -3266,6 +3279,16 @@ class MemoryEntry(BaseModel): artifact_refs: list[ArtifactReference] +class MemoryDirectoryItem(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + citation: MemoryCitation + version: Annotated[StrictInt, Field(ge=1)] + kind: StrictStr + state: MemoryEntryState + + class MemoryMutationResponse(BaseModel): model_config = ConfigDict( extra="forbid", @@ -3900,6 +3923,15 @@ class ListMemoryEntriesResponse(BaseModel): entries: list[MemoryEntry] +class QueryMemoryEntriesResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + memory_ref: Annotated[ArtifactReference | None, Field(...)] + items: Annotated[list[MemoryDirectoryItem], Field(max_length=100)] + next_cursor: Annotated[StrictStr | None, Field(max_length=4096)] + + class PrepareContextRequest(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index 221caae794..7cbdf1df7b 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -120,6 +120,8 @@ PublishRemoteSkillRequest, PutProfilePolicyRequest, QueryArtifactTagsRequest, + QueryMemoryEntriesRequest, + QueryMemoryEntriesResponse, ReadinessResponse, ReconcileRemoteSkillsRequest, ReconcileRemoteSkillsResponse, @@ -1282,6 +1284,35 @@ class AccessRequirement(BaseModel): access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), ) +QUERY_MEMORY_ENTRIES = Operation[QueryMemoryEntriesRequest, QueryMemoryEntriesResponse]( + method="POST", + path="/v1/memory/entries/query", + operation_id="query_memory_entries", + request_type=QueryMemoryEntriesRequest, + request_location="body", + path_parameters=(), + response_type=QueryMemoryEntriesResponse, + success_status=200, + summary="Query a bounded page of Memory entry identities", + tags=("memory",), + scope_mode="current", + responses={ + 200: { + "description": "One bounded page pinned to an exact Memory Revision.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + }, + 400: {"$ref": "#/components/responses/BadRequest"}, + 410: {"$ref": "#/components/responses/CursorExpired"}, + 413: {"$ref": "#/components/responses/MemoryDirectoryItemTooLarge"}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + 500: {"$ref": "#/components/responses/InternalError"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + GET_MEMORY_ENTRY = Operation[GetMemoryEntryRequest, MemoryEntry]( method="POST", path="/v1/memory/entries/get", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index 599f756efa..af6cad1197 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -1639,6 +1639,55 @@ "x-powercontext-scope-mode": "current", } }, + "/v1/memory/entries/query": { + "post": { + "tags": ["memory"], + "summary": "Query a bounded page of Memory entry identities", + "description": "Traverse compact Memory entry " + "identities in entry_id order " + "without loading entry bodies. " + "Page one pins the current Memory " + "Revision and later pages retain " + "that revision. Use this operation " + "for a bounded inventory or audit; " + "use exact entry detail for text " + "and evidence. Tag changes " + "invalidate a filtered " + "continuation and require " + "restarting from page one. An " + "empty page with a null memory_ref " + "means the Scope has no Memory.", + "operationId": "query_memory_entries", + "requestBody": { + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/QueryMemoryEntriesRequest"}} + }, + "required": True, + }, + "responses": { + "200": { + "description": "One bounded page pinned to an exact Memory Revision.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": { + "application/json": {"schema": {"$ref": "#/components/schemas/QueryMemoryEntriesResponse"}} + }, + }, + "400": {"$ref": "#/components/responses/BadRequest"}, + "410": {"$ref": "#/components/responses/CursorExpired"}, + "413": {"$ref": "#/components/responses/MemoryDirectoryItemTooLarge"}, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + "500": {"$ref": "#/components/responses/InternalError"}, + }, + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + "x-powercontext-scope-mode": "current", + } + }, "/v1/memory/entries/get": { "post": { "tags": ["memory"], @@ -7502,6 +7551,36 @@ "type": "object", "required": ["entries"], }, + "QueryMemoryEntriesRequest": { + "properties": { + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, + "include_inactive": { + "type": "boolean", + "description": "Include inactive entries from the pinned Memory Revision for explicit audit.", + "default": False, + }, + "tag_filter": {"$ref": "#/components/schemas/TagFilter"}, + "limit": {"type": "integer", "maximum": 100.0, "minimum": 1.0, "default": 50}, + "cursor": {"type": "string", "maxLength": 4096, "minLength": 1, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["scope_id"], + }, + "QueryMemoryEntriesResponse": { + "properties": { + "memory_ref": {"allOf": [{"$ref": "#/components/schemas/ArtifactReference"}], "nullable": True}, + "items": { + "items": {"$ref": "#/components/schemas/MemoryDirectoryItem"}, + "type": "array", + "maxItems": 100, + }, + "next_cursor": {"type": "string", "maxLength": 4096, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["memory_ref", "items", "next_cursor"], + }, "ListArtifactCandidatesRequest": { "properties": { "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, @@ -7537,6 +7616,17 @@ "type": "object", "required": ["citation", "version", "kind", "text", "state", "source_refs", "artifact_refs"], }, + "MemoryDirectoryItem": { + "properties": { + "citation": {"$ref": "#/components/schemas/MemoryCitation"}, + "version": {"type": "integer", "minimum": 1.0}, + "kind": {"type": "string"}, + "state": {"$ref": "#/components/schemas/MemoryEntryState"}, + }, + "additionalProperties": False, + "type": "object", + "required": ["citation", "version", "kind", "state"], + }, "MemoryMutationResponse": { "properties": { "memory": {"$ref": "#/components/schemas/ArtifactReference"}, @@ -9871,6 +9961,11 @@ "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, }, + "MemoryDirectoryItemTooLarge": { + "description": "One compact Memory directory item exceeds the public page byte budget.", + "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, + }, "NotFound": { "description": "The requested durable value was not found or is not observable.", "headers": {"X-PowerContext-Request-ID": {"$ref": "#/components/headers/RequestId"}}, diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index c8cfa7ddfd..a6850ad976 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -63,6 +63,7 @@ InvalidMemoryCandidateError, InvalidMemoryCitationError, InvalidMemoryEvidenceError, + MemoryDirectoryItemTooLargeError, MemoryEntryInactiveError, MemoryEntryNotFoundError, ) @@ -144,6 +145,7 @@ RepositoryNotFoundError, StoredPayloadConflictError, ) +from powercontext.builtin.persistence.memory_query_migration import MemoryQueryIndexUnavailableError from powercontext.builtin.persistence.skill_publications import SkillPublication from powercontext.builtin.publication import ( ArtifactPublicationApplication, @@ -212,6 +214,7 @@ HandoffResolution, InvalidRuntimeRequestError, MemoryChangesPage, + MemoryDirectoryPage, MemoryEntriesPage, MemoryEntryRecord, MemoryFlushResult, @@ -269,6 +272,7 @@ ProposeExperienceRequest as RuntimeProposeExperienceRequest, ) from powercontext.builtin.runtime import ProposeSkillRequest as RuntimeProposeSkillRequest +from powercontext.builtin.runtime import QueryMemoryEntriesRequest as RuntimeQueryMemoryEntriesRequest from powercontext.builtin.runtime import ( RejectArtifactCandidateRequest as RuntimeRejectArtifactCandidateRequest, ) @@ -496,6 +500,8 @@ PublishArtifactRequest, PublishRemoteSkillRequest, PutProfilePolicyRequest, + QueryMemoryEntriesRequest, + QueryMemoryEntriesResponse, ReadinessResponse, ReadinessStatus, ReconcileRemoteSkillsRequest, @@ -716,6 +722,7 @@ PUT_PROFILE_POLICY, QUERY_ARTIFACT_TAGS, QUERY_CODE, + QUERY_MEMORY_ENTRIES, RECONCILE_REMOTE_SKILLS, RECORD_REMOTE_SKILL_RECEIPT, RECORD_SKILL_USAGE, @@ -1156,6 +1163,8 @@ async def remember(self, request: RuntimeRememberMemoryRequest, /) -> MemoryMuta async def search(self, request: RuntimeSearchMemoryRequest, /) -> MemorySearchPage: ... + async def query(self, request: RuntimeQueryMemoryEntriesRequest, /) -> MemoryDirectoryPage: ... + async def list( self, *, include_inactive: bool = False, tag_filter: RuntimeTagFilter | None = None ) -> MemoryEntriesPage: ... @@ -1425,6 +1434,7 @@ async def unexpected_error(request: Request, error: Exception) -> JSONResponse: _add_route(app, COMMIT_HANDOFF, commit_handoff) _add_route(app, CONTINUE_HANDOFF, continue_handoff) _add_route(app, LIST_MEMORY_ENTRIES, list_memory_entries) + _add_route(app, QUERY_MEMORY_ENTRIES, query_memory_entries) _add_route(app, GET_MEMORY_ENTRY, get_memory_entry) _add_route(app, REVISE_MEMORY_ENTRY, revise_memory_entry) _add_route(app, RETIRE_MEMORY_ENTRY, retire_memory_entry) @@ -3100,6 +3110,25 @@ async def list_memory_entries( return mapping.entries_response(result) +async def query_memory_entries( + request: QueryMemoryEntriesRequest, + application: Annotated[ServerApplication, Depends(_require_application)], +) -> QueryMemoryEntriesResponse: + result = await application.memory.for_scope(request.scope_id).query( + RuntimeQueryMemoryEntriesRequest( + include_inactive=request.include_inactive, + tag_filter=( + None + if request.tag_filter is None + else RuntimeTagFilter.model_validate_json(request.tag_filter.model_dump_json()) + ), + limit=request.limit, + cursor=request.cursor, + ) + ) + return mapping.directory_response(result) + + async def get_memory_entry( request: GetMemoryEntryRequest, application: Annotated[ServerApplication, Depends(_require_application)], @@ -4318,6 +4347,7 @@ def _add_route( _COLLECTION_CONTENT_OPERATIONS = frozenset({ "search_memory", "list_memory_entries", + "query_memory_entries", "list_memory_changes", "prepare_context", "list_managed_skills", @@ -5190,6 +5220,20 @@ def _map_service_error(error: Exception) -> tuple[int, str, str, dict[str, Any] ) if isinstance(error, _RuntimeNotReadyError): return status.HTTP_503_SERVICE_UNAVAILABLE, "runtime_not_ready", "The Runtime is not ready.", None + if isinstance(error, MemoryQueryIndexUnavailableError): + return ( + status.HTTP_503_SERVICE_UNAVAILABLE, + "memory_query_index_unavailable", + "The Memory query index is not ready.", + None, + ) + if isinstance(error, MemoryDirectoryItemTooLargeError): + return ( + status.HTTP_413_CONTENT_TOO_LARGE, + "memory_directory_item_too_large", + "One Memory directory item exceeds the page byte budget.", + None, + ) base_access_error = _map_base_access_error(error) if base_access_error is not None: return base_access_error diff --git a/src/powercontext/server/mapping.py b/src/powercontext/server/mapping.py index 913bd45d90..b7424e33f8 100644 --- a/src/powercontext/server/mapping.py +++ b/src/powercontext/server/mapping.py @@ -73,6 +73,7 @@ InvalidRuntimeRequestError, MemoryChange, MemoryChangesPage, + MemoryDirectoryPage, MemoryEntriesPage, MemoryEntryInput, MemoryEntryRecord, @@ -231,6 +232,7 @@ ListMemoryChangesResponse, ListMemoryEntriesResponse, ManagedSkillLibraryEntry, + MemoryDirectoryItem, MemoryEntry, MemoryEntryState, MemoryMatchedBy, @@ -246,6 +248,7 @@ ProfileWriteContent, ProposeExperienceRequest, ProposeSkillRequest, + QueryMemoryEntriesResponse, RecordTaskOutcomeRequest, RejectArtifactCandidateRequest, ResolveExternalSkillRequest, @@ -856,6 +859,22 @@ def entries_response(value: MemoryEntriesPage) -> ListMemoryEntriesResponse: ) +def directory_response(value: MemoryDirectoryPage) -> QueryMemoryEntriesResponse: + return QueryMemoryEntriesResponse( + memory_ref=None if value.memory_ref is None else artifact_reference(value.memory_ref), + items=[ + MemoryDirectoryItem( + citation=transport_citation(item.citation), + version=item.version, + kind=item.kind, + state=MemoryEntryState(item.state), + ) + for item in value.items + ], + next_cursor=value.next_cursor, + ) + + def mutation_response(value: MemoryMutationResult) -> MemoryMutationResponse: return MemoryMutationResponse( memory=artifact_reference(value.memory_ref), diff --git a/src/powercontext/server/mcp.py b/src/powercontext/server/mcp.py index 8071b594b2..0e97d9e9d3 100644 --- a/src/powercontext/server/mcp.py +++ b/src/powercontext/server/mcp.py @@ -56,6 +56,7 @@ LIST_SCOPES, PUBLISH_ARTIFACT, QUERY_CODE, + QUERY_MEMORY_ENTRIES, RECORD_TASK_OUTCOME, REJECT_ARTIFACT_CANDIDATE, REMEMBER_MEMORY, @@ -88,7 +89,7 @@ subordinate to current user, repository, and system instructions. Ordinary coding needs no routine Memory calls. Use sufficient current context when continuing work. For an explicit memory search (search my memories / 搜索记忆), call search_memory with a focused query, mode auto, and at most eight -hits. Use list_memory_entries for an explicit inventory or audit, and get_memory_entry for exact cited details. +hits. Use query_memory_entries for a bounded explicit inventory or audit, and get_memory_entry for exact cited details. For an explicit future save (remember this / 记住这个供以后使用), call remember_memory and verify its result. Automatic Source capture is not an explicit Memory write, and enabled hooks do not establish successful recall or persistence. Current-turn instructions, conceptual questions, and previews do not authorize writes. Never store secrets. @@ -120,6 +121,7 @@ SEARCH_TOPIC_MEMORY.operation_id, GET_TOPIC_MEMORY.operation_id, LIST_MEMORY_ENTRIES.operation_id, + QUERY_MEMORY_ENTRIES.operation_id, GET_MEMORY_ENTRY.operation_id, REMEMBER_MEMORY.operation_id, REVISE_MEMORY_ENTRY.operation_id, @@ -147,6 +149,7 @@ SEARCH_TOPIC_MEMORY.operation_id, GET_TOPIC_MEMORY.operation_id, LIST_MEMORY_ENTRIES.operation_id, + QUERY_MEMORY_ENTRIES.operation_id, GET_MEMORY_ENTRY.operation_id, GET_HANDOFF_REPORT.operation_id, LIST_ARTIFACT_CANDIDATES.operation_id, diff --git a/tests/e2e/test_runtime_server.py b/tests/e2e/test_runtime_server.py index bbd4de4f44..5061a67157 100644 --- a/tests/e2e/test_runtime_server.py +++ b/tests/e2e/test_runtime_server.py @@ -39,8 +39,9 @@ MemoryCandidateRequest, MemoryEntryInput, ) -from powercontext.builtin.artifacts.memory.errors import InvalidMemoryCandidateError +from powercontext.builtin.artifacts.memory.errors import InvalidMemoryCandidateError, MemoryDirectoryItemTooLargeError from powercontext.builtin.inference import EmbeddingResult, InferenceConfigurationError +from powercontext.builtin.persistence.memory_query_migration import MemoryQueryIndexUnavailableError from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig from powercontext.builtin.runtime import ( @@ -76,6 +77,7 @@ ListMemoryEntriesRequest, PrepareContextRequest, PublishArtifactRequest, + QueryMemoryEntriesRequest, ReadinessStatus, RecordTaskOutcomeRequest, RememberMemoryRequest, @@ -835,6 +837,13 @@ async def scenario() -> None: include_inactive=True, ), ) + directory = await client.query_memory_entries( + QueryMemoryEntriesRequest( + scope_id=scope_id, + include_inactive=True, + limit=1, + ) + ) retired_search = await client.search_memory( SearchMemoryRequest( scope_id=scope_id, @@ -872,6 +881,15 @@ async def scenario() -> None: assert current.entries == [] assert audited.memory == retired.memory assert audited.entries == [retired.entry] + assert directory.memory_ref == retired.memory + assert directory.next_cursor is None + assert len(directory.items) == 1 + assert directory.items[0].model_dump() == { + "citation": retired.entry.citation.model_dump(), + "version": retired.entry.version, + "kind": retired.entry.kind, + "state": retired.entry.state, + } assert retired_search.memory == retired.memory assert retired_search.hits == [] assert retired_exact == retired.entry @@ -881,6 +899,35 @@ async def scenario() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize( + ("error", "expected_status", "expected_code"), + [ + (MemoryQueryIndexUnavailableError(), 503, "memory_query_index_unavailable"), + (MemoryDirectoryItemTooLargeError(), 413, "memory_directory_item_too_large"), + ], +) +def test_memory_directory_query_preserves_public_failure_contract( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + error: Exception, + expected_status: int, + expected_code: str, +) -> None: + async def failing_query(self, request, /): + del self, request + raise error + + monkeypatch.setattr(ScopedMemoryApplication, "query", failing_query) + app = create_server_app(settings=_server_settings(tmp_path / f"{expected_code}.db")) + + with TestClient(app) as transport: + scope_id = transport.get("/v1/scopes/default").json()["scope_id"] + response = transport.post("/v1/memory/entries/query", json={"scope_id": scope_id}) + + assert response.status_code == expected_status + assert response.json()["error"]["code"] == expected_code + + def test_runtime_conflicts_keep_http_and_sdk_error_context(tmp_path: Path) -> None: app = create_server_app(settings=_server_settings(tmp_path / "runtime.db")) diff --git a/tests/test_access_mcp.py b/tests/test_access_mcp.py index 0a3b4a8519..7f44678465 100644 --- a/tests/test_access_mcp.py +++ b/tests/test_access_mcp.py @@ -25,7 +25,7 @@ from starlette.middleware import Middleware from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile -from powercontext.builtin.runtime import MemoryEntriesPage +from powercontext.builtin.runtime import MemoryDirectoryPage, MemoryEntriesPage from powercontext.server.app import create_app from powercontext.server.authentication import StaticBearerAuthenticationProvider from powercontext.server.authz import ( @@ -59,6 +59,10 @@ async def list(self, *, include_inactive: bool = False) -> MemoryEntriesPage: del include_inactive return MemoryEntriesPage(memory_ref=None) + async def query(self, request) -> MemoryDirectoryPage: + del request + return MemoryDirectoryPage(memory_ref=None) + def test_mcp_internal_bridge_preserves_principal_and_audits_mcp_transport() -> None: async def scenario() -> None: @@ -132,11 +136,31 @@ def create_http_client( async with app.router.lifespan_context(app), Client(transport) as client: result = await client.call_tool("list_memory_entries", {"scope_id": "scope-a"}) assert result.is_error is False + query = await client.call_tool("query_memory_entries", {"scope_id": "scope-a", "limit": 1}) + assert query.is_error is False + denied_query = await client.call_tool( + "query_memory_entries", + {"scope_id": "scope-b", "limit": 1}, + raise_on_error=False, + ) + assert denied_query.is_error is True audit = await repository.list_audit(resource=ResourceRef.server()) decision = next(event for event in audit if event.operation == "list_memory_entries") assert decision.transport == "mcp" assert decision.principal == BOB assert decision.allowed is True + query_decision = next(event for event in audit if event.operation == "query_memory_entries") + assert query_decision.transport == "mcp" + assert query_decision.principal == BOB + assert query_decision.allowed is True + denied_decision = next( + event + for event in audit + if event.operation == "query_memory_entries" and event.resource == ResourceRef.scope("scope-b") + ) + assert denied_decision.transport == "mcp" + assert denied_decision.principal == BOB + assert denied_decision.allowed is False asyncio.run(scenario()) diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 872ad33b47..997f2e8bb1 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -67,6 +67,7 @@ PrepareHandoffRequest, ProposeExperienceRequest, ProposeSkillRequest, + QueryMemoryEntriesRequest, RecordTaskOutcomeRequest, ResolveExternalSkillRequest, ReviseArtifactCandidateRequest, @@ -792,6 +793,30 @@ def test_scope_pagination_declares_cursor_failures() -> None: assert responses["410"] == {"$ref": "#/components/responses/CursorExpired"} +def test_memory_directory_query_is_bounded_and_declares_cursor_failures() -> None: + contract = yaml.safe_load(CONTRACT_PATH.read_text()) + operation = contract["paths"]["/v1/memory/entries/query"]["post"] + responses = operation["responses"] + schemas = contract["components"]["schemas"] + + assert operation["operationId"] == "query_memory_entries" + assert operation["x-powercontext-access"] == { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + } + assert responses["400"] == {"$ref": "#/components/responses/BadRequest"} + assert responses["410"] == {"$ref": "#/components/responses/CursorExpired"} + assert responses["413"] == {"$ref": "#/components/responses/MemoryDirectoryItemTooLarge"} + assert schemas["QueryMemoryEntriesRequest"]["properties"]["limit"] == { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 50, + } + assert set(schemas["MemoryDirectoryItem"]["properties"]) == {"citation", "version", "kind", "state"} + assert QueryMemoryEntriesRequest(scope_id="project").limit == 50 + + def test_base_access_create_requests_leave_identity_generation_to_the_server() -> None: contract = yaml.safe_load(CONTRACT_PATH.read_text()) schemas = contract["components"]["schemas"] diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 2e5af73b59..716ad24176 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -100,6 +100,7 @@ async def inspect_components() -> tuple[list[str], int, int]: "list_dream_runs", "list_artifact_candidates", "list_memory_entries", + "query_memory_entries", "list_scopes", "publish_artifact", "query_code", From d07ae366529a2f9dae61857f8dda081d4a2c8cea Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:16:49 +0800 Subject: [PATCH 07/14] perf(memory): batch directory backfill writes --- .../en/docs/operate/memory-query-migration.md | 6 + .../1656_bounded_memory_entry_pagination.md | 32 ++- .../zh/docs/operate/memory-query-migration.md | 4 + .../1656_bounded_memory_entry_pagination.md | 27 ++- scripts/measure_memory_directory.py | 225 ++++++++++++++++++ .../persistence/memory_query_migration.py | 70 +++--- .../test_memory_query_migration.py | 140 ++++++++++- 7 files changed, 465 insertions(+), 39 deletions(-) create mode 100644 scripts/measure_memory_directory.py diff --git a/docs/en/docs/operate/memory-query-migration.md b/docs/en/docs/operate/memory-query-migration.md index e0a9ece3c6..1d5f9a476a 100644 --- a/docs/en/docs/operate/memory-query-migration.md +++ b/docs/en/docs/operate/memory-query-migration.md @@ -35,6 +35,12 @@ result before reporting success. If it stops, repeat it with the same migration ID. Do not clear the marker or directory rows; a different ID is rejected while an earlier migration is incomplete. +The batch size counts revisions, not entries inside a revision. Rebuilding one +large initial manifest therefore writes one derived directory row per entry in +that transaction. Those rows use set reads and executemany writes in chunks of +500; choose the revision batch size and maintenance window with the largest +stored manifest in mind. + Only restart writers and Server replicas after the command reports `ready: true`. The separately runnable `verify` action rechecks every authoritative revision and tag-generation row and is safe to repeat. diff --git a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md index c67c80301b..472e083e81 100644 --- a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md @@ -69,7 +69,8 @@ After authentication and authorization, the new operation returns `503 memory_qu falls back to the legacy full-manifest path. Legacy list, exact detail, search, and Memory writes remain available outside a declared maintenance window. -The operator runs an explicit offline `plan`, bounded/resumable `apply`, and `verify` workflow for SQLite or OceanBase. +The operator runs an explicit offline `plan`, revision-batched/resumable `apply`, and `verify` workflow for SQLite or +OceanBase. Only successful verification marks the feature ready. Interrupted apply remains resumable and never rewrites authoritative Memory data. Outside the maintenance window, new Memory commits maintain their directory deltas even while historical backfill is incomplete, but the query remains unavailable until full verification succeeds. @@ -110,7 +111,7 @@ legacy data or a future incompatible shape creates one item that cannot fit, the - An oversized item returns no items and no replacement cursor. - Memory and tag writes are all-or-nothing: authoritative state and related index/generation deltas commit or roll back together. -- Migration apply commits bounded checkpoints and is resumable; verify alone may mark readiness complete. +- Migration apply commits durable revision checkpoints and is resumable; verify alone may mark readiness complete. ## 6. Concurrency and resource constraints @@ -126,6 +127,27 @@ legacy data or a future incompatible shape creates one item that cannot fit, the projection rows for unchanged entries remain untouched, preserving #1709. - Backfill uses a positive batch size and durable checkpoint; startup never runs an unbounded rebuild. +### Reproducible SQLite scale evidence + +Run `uv run python scripts/measure_memory_directory.py`. The 2026-09-28 run used SQLite 3.50.4 and exercised the +production Memory write/query interfaces, then rebuilt the derived directory with `batch_size=1` revision. + +| Entries | Returned | Materialized | Item bytes | Page bytes | Directory add/close | Head row delta | Directory rows after revision batches | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 200 | 100 | 101 | 24,200 | 24,805 | 1 / 1 | 0 | 200, 201 | +| 1,000 | 100 | 101 | 24,200 | 24,805 | 1 / 1 | 0 | 1,000, 1,001 | +| 5,000 | 100 | 101 | 24,200 | 24,805 | 1 / 1 | 0 | 5,000, 5,001 | + +For all three sizes, `EXPLAIN QUERY PLAN` reported an index search on +`ix_pc_memory_entry_directory_page (scope_id, memory_artifact_id, entry_id>?)` followed by an indexed exact lookup in +`pc_memory_entry_versions`. SQLite does not report examined-row counts, so this evidence claims only 100 returned rows +and 101 rows materialized by the application, not a numeric database-examined-row bound. + +The migration batch size limits authoritative revisions, not entries inside one revision: the initial revision above +necessarily rebuilt one derived row per entry, while the second revision added one row. The migration authority uses +set reads and executemany writes in chunks of 500, avoiding per-entry database round trips without claiming constant +row work for an arbitrarily large manifest. + ## 7. Non-goals This PR does not paginate or alter `list_memory_entries`; return entry bodies in the directory operation; implement @@ -238,6 +260,6 @@ format, lint, and type checks; then `make check` and `make unit-test`. Report un separately from passes. Current status: the revision-valid directory, tag-generation invalidation, bounded runtime query, feature-scoped -migration/readiness gate, operator CLI, and public OpenAPI/Server/SDK/MCP surface are implemented with focused -persistence, contract, access, and runtime tests. The scale report, OceanBase migration evidence, and full repository -checks remain. +migration/readiness gate, operator CLI, public OpenAPI/Server/SDK/MCP surface, and reproducible SQLite scale report are +implemented with focused persistence, contract, access, and runtime tests. A live OceanBase migration contract is +available when its test service is configured; that service and the full repository checks remain to be run. diff --git a/docs/zh/docs/operate/memory-query-migration.md b/docs/zh/docs/operate/memory-query-migration.md index 4da89c0c52..2facead01e 100644 --- a/docs/zh/docs/operate/memory-query-migration.md +++ b/docs/zh/docs/operate/memory-query-migration.md @@ -29,6 +29,10 @@ powercontext server memory-query-migrate --action verify --env-file .env 并在报告成功前验证结果。中断后使用相同 migration ID 重复执行即可。不要清除 marker 或 directory 行；前一次迁移未完成时，更换 ID 会被拒绝。 +Batch size 计数的是 revision，不是一个 revision 内的 entry。因此重建大型初始 manifest 时， +该事务仍会为每个 entry 写入一条派生 directory 行。实现使用 set read 和每 500 行一批的 +executemany write；选择 revision batch size 和维护窗口时，要把已存储的最大 manifest 纳入考虑。 + 只有命令返回 `ready: true` 后才能重启 writer 和 Server 副本。独立的 `verify` action 会重新检查所有权威 revision 和 tag generation 行，可安全重复执行。 diff --git a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md index a96762eeb5..0db6e4a9ad 100644 --- a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md @@ -94,7 +94,7 @@ Server 以固定 4 MiB 预算计量编码后 directory item payload。如果下 - 认证/授权错误优先于 readiness 和 Memory 存在性。 - oversized item 不返回 item 和替换 cursor。 - Memory/tag write 与对应 index/generation delta 一起 commit 或 rollback。 -- migration apply 保存有界 checkpoint，可恢复；只有 verify 能标记 complete。 +- migration apply 保存持久 revision checkpoint，可恢复；只有 verify 能标记 complete。 ## 6. 并发与资源约束 @@ -106,6 +106,26 @@ Server 以固定 4 MiB 预算计量编码后 directory item payload。如果下 - Memory commit 只更新 pointer/state 已变的 validity row；未变 search projection 保持 #1709 的 delta 性质。 - backfill 使用正 batch size 和持久 checkpoint；startup 不执行无界 rebuild。 +### 可复现 SQLite 规模证据 + +运行 `uv run python scripts/measure_memory_directory.py`。2026-09-28 的结果使用 SQLite 3.50.4，通过生产 +Memory write/query 接口建立数据，再以 `batch_size=1` revision 重建派生 directory。 + +| Entry 数 | 返回行 | Materialized 行 | Item bytes | Page bytes | Directory 新增/关闭 | Head 行变化 | Revision batch 后 directory 行数 | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 200 | 100 | 101 | 24,200 | 24,805 | 1 / 1 | 0 | 200, 201 | +| 1,000 | 100 | 101 | 24,200 | 24,805 | 1 / 1 | 0 | 1,000, 1,001 | +| 5,000 | 100 | 101 | 24,200 | 24,805 | 1 / 1 | 0 | 5,000, 5,001 | + +三种规模的 `EXPLAIN QUERY PLAN` 都使用 +`ix_pc_memory_entry_directory_page (scope_id, memory_artifact_id, entry_id>?)` 索引搜索，再对 +`pc_memory_entry_versions` 执行索引精确查找。SQLite 不报告 examined-row 数，因此这份证据只声称返回 +100 行、应用层 materialize 101 行，不声称数据库 examined row 的数字上限。 + +Migration batch size 限制的是权威 revision 数，不是单个 revision 内的 entry 数：上表的初始 revision +必须为每个 entry 重建一行，第二个 revision 只新增一行。Migration authority 用 set read 和每 500 行一批的 +executemany write 避免 per-entry 数据库往返，但不声称任意大 manifest 具有常数 row work。 + ## 7. 非目标 本 PR 不改动 `list_memory_entries`，不在 directory 中返回 body，不实现 #1657 history pagination，不实现 @@ -211,5 +231,6 @@ review 前运行定向 OpenAPI、cursor、Memory persistence、tag、migration 不可用服务和 skipped check 必须与 pass 分开报告。 当前状态：revision-valid directory、tag generation 失效、有界 runtime query、feature-scoped -migration/readiness gate、运维 CLI 和公开 OpenAPI/Server/SDK/MCP 接口已实现，并通过定向 -persistence、contract、access 和 runtime 测试。规模报告、OceanBase migration 证据和全仓检查仍待完成。 +migration/readiness gate、运维 CLI、公开 OpenAPI/Server/SDK/MCP 接口与可复现 SQLite 规模报告已实现， +并通过定向 persistence、contract、access 和 runtime 测试。配置测试服务后可运行真实 OceanBase migration +contract；该服务与全仓检查仍待运行。 diff --git a/scripts/measure_memory_directory.py b/scripts/measure_memory_directory.py new file mode 100644 index 0000000000..b34e865b57 --- /dev/null +++ b/scripts/measure_memory_directory.py @@ -0,0 +1,225 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Measure bounded Memory directory work without model or network calls.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sqlite3 +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, TypeVar + +from sqlalchemy import delete, func, select + +from powercontext.builtin.artifacts.memory import MemoryDirectoryQuery, MemoryEntryInput +from powercontext.builtin.persistence import memory as persistence_memory +from powercontext.builtin.persistence.memory_query_migration import ( + apply_memory_query_migration, + verify_memory_query_migration, +) +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + BUILTIN_TABLES, + MEMORY_ENTRY_DIRECTORY_TABLE, + MEMORY_ENTRY_HEADS_TABLE, + MEMORY_QUERY_INDEX_SCHEMA_TABLE, + MEMORY_TAG_GENERATIONS_TABLE, +) +from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts + +DEFAULT_SIZES = (200, 1_000, 5_000) +PAGE_LIMIT = 100 +T = TypeVar("T") + + +class MeasurementError(RuntimeError): + """The measured behavior no longer satisfies the RFC's resource contract.""" + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise MeasurementError(message) + + +def _required(value: T | None, message: str) -> T: + if value is None: + raise MeasurementError(message) + return value + + +async def _count(connection, table) -> int: + return int(await connection.scalar(select(func.count()).select_from(table)) or 0) + + +async def _measure(entry_count: int) -> dict[str, Any]: + with TemporaryDirectory(prefix=f"powercontext-memory-directory-{entry_count}-") as directory: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{Path(directory) / 'scale.db'}") + return await _measure_database(entry_count, config) + + +async def _measure_database(entry_count: int, config: SQLiteConfig) -> dict[str, Any]: + runtime_config = BuiltinConfig(database=config) + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + memory = _required( + await service.remember( + memory=None, + entries=tuple( + MemoryEntryInput(kind="fact", text=f"Memory directory scale entry {index}") + for index in range(entry_count) + ), + mode="append", + ), + "scale fixture did not create a Memory", + ) + query = MemoryDirectoryQuery(limit=PAGE_LIMIT) + page = await service.query_directory(memory.artifact_id, query) + _require(page.memory_ref == memory.as_ref(), "directory page did not pin the written revision") + _require(len(page.items) == PAGE_LIMIT, "directory page did not fill the requested limit") + _require(page.next_cursor is not None, "directory page omitted continuation for remaining entries") + + statement = persistence_memory._directory_statement( + "project", + memory.artifact_id, + query, + memory.revision, + "", + ) + async with contexts.database.transaction() as connection: + materialized = (await connection.execute(statement)).mappings().all() + compiled = statement.compile( + dialect=connection.dialect, + compile_kwargs={"literal_binds": True}, + ) + plan = (await connection.exec_driver_sql(f"EXPLAIN QUERY PLAN {compiled}")).all() + directory_before = await _count(connection, MEMORY_ENTRY_DIRECTORY_TABLE) + heads_before = await _count(connection, MEMORY_ENTRY_HEADS_TABLE) + + _require(len(materialized) == PAGE_LIMIT + 1, "directory query did not materialize exactly one lookahead row") + target = (await service.entries(memory))[0] + revised = _required( + await service.remember( + memory=memory, + entries=( + MemoryEntryInput( + entry=target, + kind=target.kind, + text=f"{target.text} revised", + ), + ), + mode="append", + ), + "single-entry revision produced no Memory revision", + ) + async with contexts.database.transaction() as connection: + directory_after = await _count(connection, MEMORY_ENTRY_DIRECTORY_TABLE) + heads_after = await _count(connection, MEMORY_ENTRY_HEADS_TABLE) + closed_rows = int( + await connection.scalar( + select(func.count()) + .select_from(MEMORY_ENTRY_DIRECTORY_TABLE) + .where(MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_to_revision == revised.revision) + ) + or 0 + ) + + _require(directory_after - directory_before == 1, "single-entry revision added more than one directory row") + _require(heads_before == heads_after == entry_count, "single-entry revision changed the head-row cardinality") + _require(closed_rows == 1, "single-entry revision did not close exactly one directory row") + + # Rebuild the derived state with one revision per transaction-sized batch. + async with ( + SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile, + profile.database.transaction() as connection, + ): + await connection.execute(delete(MEMORY_ENTRY_DIRECTORY_TABLE)) + await connection.execute(delete(MEMORY_TAG_GENERATIONS_TABLE)) + await connection.execute(delete(MEMORY_QUERY_INDEX_SCHEMA_TABLE)) + + migration_processed: list[int] = [] + migration_directory_rows: list[int] = [] + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile: + while True: + async with profile.database.transaction() as connection: + progress = await apply_memory_query_migration( + connection, + migration_id=f"scale-{entry_count}", + batch_size=1, + ) + if progress.processed: + migration_processed.append(progress.processed) + migration_directory_rows.append(await _count(connection, MEMORY_ENTRY_DIRECTORY_TABLE)) + if progress.phase == "verify": + break + _require(not progress.complete, "migration unexpectedly completed without verification") + async with profile.database.transaction() as connection: + verification = await verify_memory_query_migration(connection) + + _require(migration_processed == [1, 1], "batch_size=1 did not process one revision per batch") + _require( + migration_directory_rows == [entry_count, entry_count + 1], + "migration did not rebuild the initial manifest followed by one changed entry", + ) + _require(verification.ready, "rebuilt directory did not verify") + _require(verification.checked_revisions == 2, "verification did not inspect both Memory revisions") + + return { + "entries": entry_count, + "page_limit": PAGE_LIMIT, + "returned_rows": len(page.items), + "materialized_rows": len(materialized), + "encoded_item_bytes": sum(len(item.model_dump_json().encode("utf-8")) for item in page.items), + "encoded_page_bytes": len(page.model_dump_json().encode("utf-8")), + "sqlite_query_plan": [str(row[3]) for row in plan], + "write_delta": { + "directory_rows_added": directory_after - directory_before, + "directory_rows_closed": closed_rows, + "head_row_delta": heads_after - heads_before, + }, + "migration": { + "batch_size_revisions": 1, + "processed_revisions_per_batch": migration_processed, + "directory_rows_after_each_batch": migration_directory_rows, + "verified_revisions": verification.checked_revisions, + }, + } + + +async def _run(sizes: tuple[int, ...]) -> dict[str, Any]: + return { + "environment": {"backend": "SQLite", "sqlite_version": sqlite3.sqlite_version}, + "measurements": [await _measure(size) for size in sizes], + "limits": { + "database_rows_examined": ("not reported: SQLite EXPLAIN QUERY PLAN does not provide an examined-row count") + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("sizes", nargs="*", type=int, default=DEFAULT_SIZES) + arguments = parser.parse_args() + sizes = tuple(arguments.sizes) + if not sizes or any(size < PAGE_LIMIT + 1 for size in sizes): + parser.error(f"every size must be at least {PAGE_LIMIT + 1}") + print(json.dumps(asyncio.run(_run(sizes)), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/powercontext/builtin/persistence/memory_query_migration.py b/src/powercontext/builtin/persistence/memory_query_migration.py index 696c4c9571..dd9e9b63e8 100644 --- a/src/powercontext/builtin/persistence/memory_query_migration.py +++ b/src/powercontext/builtin/persistence/memory_query_migration.py @@ -33,6 +33,7 @@ from powercontext.builtin.artifacts.memory import Memory, MemoryContent from powercontext.builtin.persistence.codec import load_model, stored_bytes +from powercontext.builtin.persistence.database import SELECTION_BATCH_SIZE from powercontext.builtin.persistence.errors import PersistenceError from powercontext.builtin.persistence.tables import ( ARTIFACT_HEADS_TABLE, @@ -384,58 +385,67 @@ async def _backfill_revision(connection: AsyncConnection, row: Mapping[Any, Any] } ) current = {entry_id: (item.entry_version_id, item.state) for entry_id, item in manifest.items()} - changed_ids = { + changed_ids = sorted( entry_id for entry_id in previous.keys() | current.keys() if previous.get(entry_id) != current.get(entry_id) - } + ) - for entry_id in sorted(changed_ids): + closed_ids = [entry_id for entry_id in changed_ids if entry_id in previous] + for offset in range(0, len(closed_ids), SELECTION_BATCH_SIZE): + batch = closed_ids[offset : offset + SELECTION_BATCH_SIZE] await connection.execute( update(MEMORY_ENTRY_DIRECTORY_TABLE) .where( MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == scope_id, MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == artifact_id, - MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id == entry_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id.in_(batch), MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision < revision, MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_to_revision.is_(None), ) .values(valid_to_revision=revision) ) - item = manifest.get(entry_id) - if item is None: - continue - existing = ( - ( - await connection.execute( - select(MEMORY_ENTRY_DIRECTORY_TABLE).where( - MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == scope_id, - MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == artifact_id, - MEMORY_ENTRY_DIRECTORY_TABLE.c.entry_id == entry_id, - MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision == revision, - ) + + existing_rows = ( + ( + await connection.execute( + select(MEMORY_ENTRY_DIRECTORY_TABLE).where( + MEMORY_ENTRY_DIRECTORY_TABLE.c.scope_id == scope_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.memory_artifact_id == artifact_id, + MEMORY_ENTRY_DIRECTORY_TABLE.c.valid_from_revision == revision, ) ) - .mappings() - .one_or_none() ) + .mappings() + .all() + ) + existing_by_entry = {str(existing["entry_id"]): existing for existing in existing_rows} + inserted: list[dict[str, Any]] = [] + for entry_id in changed_ids: + item = manifest.get(entry_id) + if item is None: + continue + existing = existing_by_entry.get(entry_id) expected = { "entry_version_id": item.entry_version_id, "state": item.state, } if existing is None: - await connection.execute( - insert(MEMORY_ENTRY_DIRECTORY_TABLE).values( - scope_id=scope_id, - family=Memory.family, - memory_artifact_id=artifact_id, - entry_id=item.entry_id, - entry_version_id=item.entry_version_id, - state=item.state, - valid_from_revision=revision, - valid_to_revision=None, - ) - ) + inserted.append({ + "scope_id": scope_id, + "family": Memory.family, + "memory_artifact_id": artifact_id, + "entry_id": item.entry_id, + "entry_version_id": item.entry_version_id, + "state": item.state, + "valid_from_revision": revision, + "valid_to_revision": None, + }) elif any(existing[key] != value for key, value in expected.items()): raise MemoryQueryIndexUnavailableError("conflict") + for offset in range(0, len(inserted), SELECTION_BATCH_SIZE): + await connection.execute( + insert(MEMORY_ENTRY_DIRECTORY_TABLE), + inserted[offset : offset + SELECTION_BATCH_SIZE], + ) generation = await connection.scalar( select(MEMORY_TAG_GENERATIONS_TABLE.c.generation).where( diff --git a/tests/builtin/persistence/test_memory_query_migration.py b/tests/builtin/persistence/test_memory_query_migration.py index ca5ec8ca1c..ab0dfe137c 100644 --- a/tests/builtin/persistence/test_memory_query_migration.py +++ b/tests/builtin/persistence/test_memory_query_migration.py @@ -15,9 +15,13 @@ from __future__ import annotations import asyncio +import os +from uuid import uuid4 import pytest -from sqlalchemy import delete, select, update +from pydantic import SecretStr +from sqlalchemy import delete, event, func, select, update +from sqlalchemy.engine import make_url from powercontext.builtin.artifacts.memory import MemoryDirectoryQuery, MemoryEntryInput from powercontext.builtin.persistence.memory_query_migration import ( @@ -26,6 +30,7 @@ plan_memory_query_migration, verify_memory_query_migration, ) +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import ( BUILTIN_TABLES, @@ -35,6 +40,8 @@ ) from powercontext.builtin.runtime import BuiltinConfig, open_builtin_contexts +LIVE_OCEANBASE_URL = os.environ.get("POWERCONTEXT_TEST_OCEANBASE_URL") + def test_existing_memory_requires_resumable_verified_directory_backfill(tmp_path) -> None: async def scenario() -> None: @@ -176,3 +183,134 @@ async def scenario() -> None: } asyncio.run(scenario()) + + +def test_memory_query_migration_batches_directory_io_for_a_large_revision(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'memory-query-batch.db'}") + async with open_builtin_contexts(BuiltinConfig(database=config)) as contexts: + service = (await contexts.get("project")).artifacts.memory + memory = await service.remember( + memory=None, + entries=tuple(MemoryEntryInput(kind="fact", text=f"Migration entry {index}") for index in range(200)), + mode="append", + ) + assert memory is not None + + async with SQLiteProfile.open(config, tables=BUILTIN_TABLES) as profile: + async with profile.database.transaction() as connection: + await connection.execute(delete(MEMORY_ENTRY_DIRECTORY_TABLE)) + await connection.execute(delete(MEMORY_TAG_GENERATIONS_TABLE)) + await connection.execute(delete(MEMORY_QUERY_INDEX_SCHEMA_TABLE)) + + statements: list[str] = [] + + def record_statement(_connection: object, _cursor: object, statement: str, *_rest: object) -> None: + statements.append(statement) + + engine = profile.database.engine.sync_engine + event.listen(engine, "before_cursor_execute", record_statement) + try: + async with profile.database.transaction() as connection: + progress = await apply_memory_query_migration( + connection, + migration_id="test-batched-1656", + batch_size=1, + ) + directory_rows = int( + await connection.scalar(select(func.count()).select_from(MEMORY_ENTRY_DIRECTORY_TABLE)) or 0 + ) + finally: + event.remove(engine, "before_cursor_execute", record_statement) + + directory_io = [ + statement + for statement in statements + if MEMORY_ENTRY_DIRECTORY_TABLE.name in statement + and statement.lstrip().upper().startswith(("SELECT", "INSERT", "UPDATE")) + ] + assert progress.processed == 1 + assert directory_rows == 200 + # One set read plus one executemany insert; work must not add a SQL + # round trip for every entry in the authoritative revision. + assert len(directory_io) == 3 # includes the final count used by this test + + asyncio.run(scenario()) + + +@pytest.mark.skipif( + not LIVE_OCEANBASE_URL, + reason=( + "set POWERCONTEXT_TEST_OCEANBASE_URL to an OceanBase MySQL-mode URL with database creation and deletion " + "privileges" + ), +) +def test_live_oceanbase_memory_query_migration_gates_and_restores_directory() -> None: + async def scenario() -> None: + assert LIVE_OCEANBASE_URL is not None + database_name = f"pc_test_{uuid4().hex}" + database_created = False + server_config = OceanBaseConfig(url=SecretStr(LIVE_OCEANBASE_URL)) + async with OceanBaseProfile.open(server_config, tables=()) as server_profile: + try: + async with server_profile.database.transaction() as connection: + await connection.exec_driver_sql(f"CREATE DATABASE `{database_name}`") + database_created = True + test_url = ( + make_url(LIVE_OCEANBASE_URL).set(database=database_name).render_as_string(hide_password=False) + ) + database_config = OceanBaseConfig(url=SecretStr(test_url)) + runtime_config = BuiltinConfig(database=database_config) + + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + memory = await service.remember( + memory=None, + entries=( + MemoryEntryInput(kind="fact", text="Keep legacy reads available."), + MemoryEntryInput(kind="decision", text="Backfill this directory on OceanBase."), + ), + mode="append", + ) + assert memory is not None + + async with ( + OceanBaseProfile.open(database_config, tables=BUILTIN_TABLES) as profile, + profile.database.transaction() as connection, + ): + await connection.execute(delete(MEMORY_ENTRY_DIRECTORY_TABLE)) + await connection.execute(delete(MEMORY_TAG_GENERATIONS_TABLE)) + await connection.execute(delete(MEMORY_QUERY_INDEX_SCHEMA_TABLE)) + + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + assert len(await service.entries(memory)) == 2 + with pytest.raises(MemoryQueryIndexUnavailableError): + await service.query_directory(memory.artifact_id, MemoryDirectoryQuery()) + + async with OceanBaseProfile.open(database_config, tables=BUILTIN_TABLES) as profile: + while True: + async with profile.database.transaction() as connection: + progress = await apply_memory_query_migration( + connection, + migration_id="test-oceanbase-1656", + batch_size=1, + ) + if progress.phase == "verify": + break + async with profile.database.transaction() as connection: + verified = await verify_memory_query_migration(connection) + assert verified.ready + assert verified.checked_revisions == 1 + + async with open_builtin_contexts(runtime_config) as contexts: + service = (await contexts.get("project")).artifacts.memory + page = await service.query_directory(memory.artifact_id, MemoryDirectoryQuery(limit=10)) + assert page.memory_ref == memory.as_ref() + assert len(page.items) == 2 + finally: + if database_created: + async with server_profile.database.transaction() as connection: + await connection.exec_driver_sql(f"DROP DATABASE `{database_name}`") + + asyncio.run(scenario()) From 902ad659a7b17fe879d5693d1e3b775bebfd4602 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:17:45 +0800 Subject: [PATCH 08/14] docs(integrations): register memory query tool --- integrations/capabilities.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integrations/capabilities.toml b/integrations/capabilities.toml index 5c5292f19d..145a1464b7 100644 --- a/integrations/capabilities.toml +++ b/integrations/capabilities.toml @@ -51,6 +51,9 @@ capabilities = ["memory_read"] id = "list_memory_entries" capabilities = ["memory_read"] [[toolsets.tools]] +id = "query_memory_entries" +capabilities = ["memory_read"] +[[toolsets.tools]] id = "get_memory_entry" capabilities = ["memory_read"] [[toolsets.tools]] From 010304ca7bb2556e39ee354b44113cbcae940320 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:23:52 +0800 Subject: [PATCH 09/14] fix(integrations): align memory query support metadata --- docs/en/docs/operate/troubleshoot.md | 4 ++-- docs/zh/docs/operate/troubleshoot.md | 4 ++-- integrations/codex/plugins/powercontext/hooks/bind_tools.py | 1 + tests/e2e/test_mcp_transport.py | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/en/docs/operate/troubleshoot.md b/docs/en/docs/operate/troubleshoot.md index 07d931b60d..4423889689 100644 --- a/docs/en/docs/operate/troubleshoot.md +++ b/docs/en/docs/operate/troubleshoot.md @@ -269,7 +269,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_memory_query_index_schema,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ -f ``` @@ -285,7 +285,7 @@ so the previous database remains available for recovery: ```bash obloader -D --csv \ - --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_memory_entry_directory,pc_memory_tag_generations,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/docs/zh/docs/operate/troubleshoot.md b/docs/zh/docs/operate/troubleshoot.md index 4aba608a5f..9d751bbd0e 100644 --- a/docs/zh/docs/operate/troubleshoot.md +++ b/docs/zh/docs/operate/troubleshoot.md @@ -260,7 +260,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ + --table 'pc_scopes,pc_source_journal_heads,pc_sources,pc_artifacts,pc_memory_query_index_schema,pc_source_cursors,pc_artifact_processing_leases,pc_artifact_processing_binding_states,pc_artifact_processing_pending,pc_artifact_processing_auto_wave_targets,pc_artifact_processing_sequences,pc_artifact_processing_intents,pc_topic_memory_processing_targets,pc_artifact_processing_schema,pc_artifact_processing_migration_receipts,pc_topic_memory_work_budgets,pc_topic_memory_retrieval_shape,pc_connector_checkpoints,pc_source_definition_manifests,pc_external_skill_registrations,pc_skill_packages,pc_agent_skill_targets,pc_skill_publications,pc_model_usage_daily,pc_recall_token_daily,pc_receipt_migration_review' \ -f ``` @@ -276,7 +276,7 @@ collation，但不会包含数据库 URL 或凭据。 ```bash obloader -D --csv \ - --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ + --table 'pc_artifact_candidate_heads,pc_topic_memory_active_topics,pc_topic_memory_active_chunks,pc_memory_entry_heads,pc_memory_entry_directory,pc_memory_tag_generations,pc_artifact_tags,pc_recurrence_match,pc_recurrence_observation' \ -f ``` diff --git a/integrations/codex/plugins/powercontext/hooks/bind_tools.py b/integrations/codex/plugins/powercontext/hooks/bind_tools.py index 76cec95abb..fd0dec838b 100644 --- a/integrations/codex/plugins/powercontext/hooks/bind_tools.py +++ b/integrations/codex/plugins/powercontext/hooks/bind_tools.py @@ -75,6 +75,7 @@ "propose_skill", "propose_skill_package", "publish_remote_skill", + "query_memory_entries", "query_code", "record_skill_usage", "record_task_outcome", diff --git a/tests/e2e/test_mcp_transport.py b/tests/e2e/test_mcp_transport.py index 104cf90587..47af961627 100644 --- a/tests/e2e/test_mcp_transport.py +++ b/tests/e2e/test_mcp_transport.py @@ -224,6 +224,7 @@ async def exercise_tools() -> set[str]: "list_memory_entries", "list_scopes", "publish_artifact", + "query_memory_entries", "query_code", "record_task_outcome", "resolve_scope_binding", From b9f8ea6a9280d82874b9b70f75a8e299630d6855 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:31:46 +0800 Subject: [PATCH 10/14] docs(rfc): finalize memory pagination design --- docs/en/rfcs/1656_bounded_memory_entry_pagination.md | 12 ++++-------- docs/zh/rfcs/1656_bounded_memory_entry_pagination.md | 12 ++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md index 472e083e81..eb94489aca 100644 --- a/docs/en/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/en/rfcs/1656_bounded_memory_entry_pagination.md @@ -4,7 +4,7 @@ - Related work: [#1657](https://github.com/oceanbase/powercontext/issues/1657), [#1709](https://github.com/oceanbase/powercontext/pull/1709), and [#1718](https://github.com/oceanbase/powercontext/issues/1718) -- Status: core design accepted in principle; implementation in progress +- Status: ready for review # Summary @@ -252,14 +252,10 @@ complete. Existing databases remain incomplete until offline verification confir | Legacy unchanged | legacy list path | existing suite plus compatibility regression | | Resource claims | persistence | reproducible scale report | -## 6. Verification plan and current status +## 6. Verification Before review, run focused OpenAPI, cursor, Memory persistence, tag, migration, Server, access-control, and client tests; SQLite scale measurements; OceanBase evidence when its service is available; generated-code, contract, documentation, format, lint, and type checks; then `make check` and `make unit-test`. Report unavailable services and skipped checks -separately from passes. - -Current status: the revision-valid directory, tag-generation invalidation, bounded runtime query, feature-scoped -migration/readiness gate, operator CLI, public OpenAPI/Server/SDK/MCP surface, and reproducible SQLite scale report are -implemented with focused persistence, contract, access, and runtime tests. A live OceanBase migration contract is -available when its test service is configured; that service and the full repository checks remain to be run. +separately from passes. The live OceanBase migration contract is conditionally runnable when its isolated test service +is configured; absence of that service is a skip, not a pass. diff --git a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md index 0db6e4a9ad..2c744efe85 100644 --- a/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md +++ b/docs/zh/rfcs/1656_bounded_memory_entry_pagination.md @@ -4,7 +4,7 @@ - 关联工作：[#1657](https://github.com/oceanbase/powercontext/issues/1657)、 [#1709](https://github.com/oceanbase/powercontext/pull/1709) 和 [#1718](https://github.com/oceanbase/powercontext/issues/1718) -- 状态：核心设计原则已达成共识；实现进行中 +- 状态：可供 review # 摘要 @@ -224,13 +224,9 @@ verify 确认派生覆盖率和精确身份前保持 incomplete。 | 旧行为不变 | legacy list path | 现有 suite + compatibility regression | | 资源声称 | persistence | 可复现 scale report | -## 6. 验证计划与当前状态 +## 6. 验证 review 前运行定向 OpenAPI、cursor、Memory persistence、tag、migration、Server、access-control 和 client 测试；SQLite 规模测量； 可用时的 OceanBase 证据；generated-code、contract、docs、format、lint、type check；以及 `make check` 和 `make unit-test`。 -不可用服务和 skipped check 必须与 pass 分开报告。 - -当前状态：revision-valid directory、tag generation 失效、有界 runtime query、feature-scoped -migration/readiness gate、运维 CLI、公开 OpenAPI/Server/SDK/MCP 接口与可复现 SQLite 规模报告已实现， -并通过定向 persistence、contract、access 和 runtime 测试。配置测试服务后可运行真实 OceanBase migration -contract；该服务与全仓检查仍待运行。 +不可用服务和 skipped check 必须与 pass 分开报告。真实 OceanBase migration contract 在配置隔离测试服务后 +可条件运行；未配置该服务是 skip，不是 pass。 From 616affb0f9f71e52ff190dd5991c66d2a6cf6a27 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:18:46 +0800 Subject: [PATCH 11/14] feat(memory): add compact Atomic directory with unified migration --- docs/en/docs/workflows/atomic-memory.md | 46 + docs/zh/docs/workflows/atomic-memory.md | 41 + integrations/capabilities.toml | 3 + .../plugins/powercontext/hooks/bind_tools.py | 1 + .../powercontext/src/operations.generated.ts | 1 + .../powercontext/src/operations.generated.ts | 1 + .../powercontext/src/operations.generated.ts | 1 + openapi/powercontext.yaml | 154 ++++ src/powercontext/builtin/artifacts/merge.py | 2 +- .../persistence/artifact_governance.py | 17 + .../builtin/persistence/artifacts.py | 29 +- .../builtin/persistence/atomic_memory.py | 11 +- .../persistence/atomic_memory_directory.py | 390 ++++++++ .../builtin/persistence/migrations/bundle.py | 65 +- .../persistence/migrations/deployment.py | 96 +- .../builtin/persistence/migrations/models.py | 1 + .../builtin/persistence/migrations/mysql.py | 2 + .../migrations/resources/manifest.json | 100 +- .../schemas/atomic_directory_sqlite_v1.json | 154 ++++ .../resources/tasks/atomic_directory_v1.py | 158 ++++ .../versions/p0004_sqlite_business.py | 41 + .../versions/p0005_atomic_directory.py | 33 + .../builtin/persistence/migrations/sqlite.py | 70 +- .../builtin/persistence/sqlite/profile.py | 4 + src/powercontext/builtin/persistence/tags.py | 7 +- .../builtin/runtime/application.py | 4 + .../builtin/runtime/atomic_memory.py | 83 +- .../builtin/runtime/composition.py | 13 + .../builtin/runtime/relational.py | 5 +- src/powercontext/client/client.py | 6 + src/powercontext/http/__init__.py | 6 + src/powercontext/http/_generated/models.py | 33 + .../http/_generated/operations.py | 29 + src/powercontext/http/_generated/schema.py | 146 +++ src/powercontext/server/app.py | 35 + src/powercontext/server/atomic_memory.py | 21 +- src/powercontext/server/authz/casbin.py | 1 + src/powercontext/server/authz/repository.py | 109 ++- src/powercontext/server/authz/service.py | 65 +- src/powercontext/server/database_migration.py | 32 +- src/powercontext/server/mcp.py | 7 +- .../test_atomic_memory_directory.py | 860 ++++++++++++++++++ tests/e2e/test_atomic_memory_directory.py | 508 +++++++++++ .../test_atomic_memory_directory_migration.py | 383 ++++++++ tests/e2e/test_mcp_transport.py | 1 + tests/test_mcp.py | 2 + 46 files changed, 3716 insertions(+), 61 deletions(-) create mode 100644 src/powercontext/builtin/persistence/atomic_memory_directory.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_directory_sqlite_v1.json create mode 100644 src/powercontext/builtin/persistence/migrations/resources/tasks/atomic_directory_v1.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0004_sqlite_business.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_directory.py create mode 100644 tests/builtin/persistence/test_atomic_memory_directory.py create mode 100644 tests/e2e/test_atomic_memory_directory.py create mode 100644 tests/e2e/test_atomic_memory_directory_migration.py diff --git a/docs/en/docs/workflows/atomic-memory.md b/docs/en/docs/workflows/atomic-memory.md index 8f8e998b68..847b3716e2 100644 --- a/docs/en/docs/workflows/atomic-memory.md +++ b/docs/en/docs/workflows/atomic-memory.md @@ -111,6 +111,52 @@ both accept `kind`, `tags` and `tag_match: "all" | "any"`, with limits of 1–10 [Artifact tag API](manage-artifact-tags.md) with `{type: "artifact", family: "atomic-memory", artifact_id: "M"}` for new targets. +## Compact directory pagination + +To browse memory identities and states, call `POST /v1/atomic-memory/directory/query`: + +```json +{"scope_id": "S", "states": ["active"], "kind": "decision", "limit": 50} +``` + +The response contains `scope_id`, `items` and `next_cursor`. Each item contains only the exact `artifact` reference, +`kind`, `state`, `state_version` and `merged_into_id`; it loads no content, sources or history. Results are ordered by +`artifact.artifact_id` ascending. Read the returned exact revision when content is needed, so a later replacement +does not change the meaning of this page. + +Keep the Scope, principal, filters and limit unchanged and copy `next_cursor` into the next request's `cursor`. +Creation, replacement, lifecycle changes, merge, restore or effective tag changes invalidate older cursors. +On `400 invalid_cursor` or `410 cursor_expired`, discard the continuation and restart at page one. This is a current +directory; it does not guarantee that a traversal completes during continuous writes. The default limit is 50 and +maximum is 100. `tags` accepts at most 16 labels of at most 64 characters each, with `tag_match: "all" | "any"`. +Every page rechecks access. Revocation prevents continuation; sharing one Artifact does not grant directory access. + +The HTTP SDK and MCP operation is `query_atomic_memory_directory`; the Python scoped method is `query_directory`. +Unmanaged deployments return `503 atomic_memory_directory_unavailable`. The optional registered target supports +persistent SQLite, without an embedding extension, and the default business layout created by the current program. +Stop all writers and use the same deployment environment file: + +This target uses the default evidence directory next to the database; a custom `--evidence-dir` is not supported. + +```bash +powercontext server db-migrate plan --target atomic-memory-directory --env-file deployment.env +powercontext server db-migrate apply --target atomic-memory-directory --env-file deployment.env +powercontext server db-migrate verify --target atomic-memory-directory --env-file deployment.env +``` + +The explicit migration adds directory metadata and backfills only exact current Atomic content. Restart the matching +program after successful verification; normal startup does not add or backfill directory fields. Other modules retain +their existing initialization behavior. Completing legacy Memory migration alone does not establish this target's +readiness: unregistered archive objects, historical layouts and vector configurations are rejected. Inspection and +verification do not repair data. Interrupted maintenance retains its original recovery point and blocks business restart. + +Directory access supports trusted local reads or native Builtin/Casbin Scope-read configurations on the same SQLite +database. Formal ownership is still checked across the entire Scope and is never guessed during migration. Unsupported access configurations +return `503 atomic_memory_directory_authorization_unavailable`. An item exceeding the response budget returns +`413 atomic_memory_directory_item_too_large`, without a partially successful page. +Restore only under stopped writes, with matching program/configuration and preserved recovery evidence. Rotate the +existing cursor signing secret before resuming service so cursors from before database restoration are rejected. + ## Merge and restore `POST /v1/atomic-memory/merges` requires at least two exact active input references and state versions, plus result content: diff --git a/docs/zh/docs/workflows/atomic-memory.md b/docs/zh/docs/workflows/atomic-memory.md index e6ec5f50c2..ba5f27453a 100644 --- a/docs/zh/docs/workflows/atomic-memory.md +++ b/docs/zh/docs/workflows/atomic-memory.md @@ -107,6 +107,47 @@ Atomic Memory 将每条事实、偏好或决策保存为独立的 `atomic-memory `tag_match: "all" | "any"`，条数范围为 1–100。标签使用 [Artifact 标签接口](manage-artifact-tags.md)，新目标为 `{type: "artifact", family: "atomic-memory", artifact_id: "M"}`。 +## 紧凑目录分页 + +只需浏览记忆身份和状态时，调用 `POST /v1/atomic-memory/directory/query`： + +```json +{"scope_id": "S", "states": ["active"], "kind": "decision", "limit": 50} +``` + +响应包含 `scope_id`、`items` 和 `next_cursor`。每项只有精确 `artifact` 引用、`kind`、`state`、 +`state_version` 与 `merged_into_id`，不加载正文、来源或历史。按 `artifact.artifact_id` 升序返回； +需要正文时，用返回引用调用精确 revision 读取接口，避免将后来修订的正文当成本页结果。 + +保持 Scope、主体、筛选和 limit 相同，将 `next_cursor` 原样放入下一次请求的 `cursor`。 +创建、修订、生命周期、合并、恢复或有效标签变化会使旧 cursor 失效；收到 `400 invalid_cursor` +或 `410 cursor_expired` 时丢弃续页位置，从第一页重新读取。这是当前目录，不保证持续写入期间完成一次遍历。 +默认每页 50 项，最大 100；`tags` 最多 16 个、每个最多 64 字符，支持 `tag_match: "all" | "any"`。 +每页重新检查权限，撤权后不能继续读取；单条 Artifact 分享不授予目录浏览权。 + +HTTP SDK 和 MCP 均使用 `query_atomic_memory_directory`；Python scoped 接口为 `query_directory`。 +未纳管目录的部署返回 `503 atomic_memory_directory_unavailable`。首版可选目标支持持久 SQLite、无 embedding +向量扩展，以及当前程序创建的默认业务结构。停止所有写入者后，使用同一部署环境文件： + +这个目标使用数据库旁的默认证据目录，首版不支持自定义 `--evidence-dir`。 + +```bash +powercontext server db-migrate plan --target atomic-memory-directory --env-file deployment.env +powercontext server db-migrate apply --target atomic-memory-directory --env-file deployment.env +powercontext server db-migrate verify --target atomic-memory-directory --env-file deployment.env +``` + +显式迁移增加目录字段，只回填当前 Atomic 精确正文中的类型。验证成功后启动匹配的新程序；普通启动不补目录 +字段、不回填目录。其他模块保留原有初始化行为。旧 Memory 迁移成功不能单独证明这个目标就绪：未登记的归档 +对象、历史结构和向量配置仍会被拒绝。检查和验证不修复数据；中断维护保留原恢复点，并阻止业务重新启动。 + +目录支持可信本地读取或同一 SQLite 数据库的原生 Builtin/Casbin Scope-read 配置。整个 Scope 的正式 owner +仍按共享规则检查，迁移不猜测或补造 owner。不支持的权限配置返回 +`503 atomic_memory_directory_authorization_unavailable`。单项超出响应预算返回 +`413 atomic_memory_directory_item_too_large`，不会返回部分成功页。 +恢复数据库时保持停写，使用匹配的程序/配置并保留恢复证据；恢复后先轮换既有 cursor 签名密钥，再恢复服务， +让数据库恢复前的旧 cursor 全部失效。 + ## 合并与恢复 `POST /v1/atomic-memory/merges` 至少需要两个 active 输入的精确引用和状态版本，以及结果正文： diff --git a/integrations/capabilities.toml b/integrations/capabilities.toml index 44f7f5fb07..e19b8c7407 100644 --- a/integrations/capabilities.toml +++ b/integrations/capabilities.toml @@ -70,6 +70,9 @@ capabilities = ["memory_read"] id = "list_atomic_memories" capabilities = ["memory_read"] [[toolsets.tools]] +id = "query_atomic_memory_directory" +non_profile_reason = "Optional compact directory browsing requires the registered persistent SQLite directory migration and qualified Scope access; it is not a mandatory host profile operation." +[[toolsets.tools]] id = "search_atomic_memory" capabilities = ["memory_read"] [[toolsets.tools]] diff --git a/integrations/codex/plugins/powercontext/hooks/bind_tools.py b/integrations/codex/plugins/powercontext/hooks/bind_tools.py index f1f7b2f317..ed9ec129a4 100644 --- a/integrations/codex/plugins/powercontext/hooks/bind_tools.py +++ b/integrations/codex/plugins/powercontext/hooks/bind_tools.py @@ -69,6 +69,7 @@ "import_external_skill", "list_artifact_candidates", "list_atomic_memories", + "query_atomic_memory_directory", "list_external_skills", "list_managed_skills", "list_memory_entries", diff --git a/integrations/dsh/plugins/powercontext/src/operations.generated.ts b/integrations/dsh/plugins/powercontext/src/operations.generated.ts index 34c0b7dd54..b8da435a01 100644 --- a/integrations/dsh/plugins/powercontext/src/operations.generated.ts +++ b/integrations/dsh/plugins/powercontext/src/operations.generated.ts @@ -17,6 +17,7 @@ // generated from openapi/powercontext.yaml; do not edit. export const OPERATIONS = { + query_atomic_memory_directory: { method: 'POST', path: '/v1/atomic-memory/directory/query', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_atomic_memories: { method: 'POST', path: '/v1/atomic-memory/list', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_atomic_memory: { method: 'POST', path: '/v1/atomic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, merge_atomic_memories: { method: 'POST', path: '/v1/atomic-memory/merges', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/opencode/plugins/powercontext/src/operations.generated.ts b/integrations/opencode/plugins/powercontext/src/operations.generated.ts index 34c0b7dd54..b8da435a01 100644 --- a/integrations/opencode/plugins/powercontext/src/operations.generated.ts +++ b/integrations/opencode/plugins/powercontext/src/operations.generated.ts @@ -17,6 +17,7 @@ // generated from openapi/powercontext.yaml; do not edit. export const OPERATIONS = { + query_atomic_memory_directory: { method: 'POST', path: '/v1/atomic-memory/directory/query', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_atomic_memories: { method: 'POST', path: '/v1/atomic-memory/list', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_atomic_memory: { method: 'POST', path: '/v1/atomic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, merge_atomic_memories: { method: 'POST', path: '/v1/atomic-memory/merges', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/integrations/pi/plugins/powercontext/src/operations.generated.ts b/integrations/pi/plugins/powercontext/src/operations.generated.ts index 34c0b7dd54..b8da435a01 100644 --- a/integrations/pi/plugins/powercontext/src/operations.generated.ts +++ b/integrations/pi/plugins/powercontext/src/operations.generated.ts @@ -17,6 +17,7 @@ // generated from openapi/powercontext.yaml; do not edit. export const OPERATIONS = { + query_atomic_memory_directory: { method: 'POST', path: '/v1/atomic-memory/directory/query', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, list_atomic_memories: { method: 'POST', path: '/v1/atomic-memory/list', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, search_atomic_memory: { method: 'POST', path: '/v1/atomic-memory/search', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, merge_atomic_memories: { method: 'POST', path: '/v1/atomic-memory/merges', location: "body", scopeMode: 'current', pathParameters: [], queryParams: [], headerParams: [], successStatuses: [200], emptyStatuses: [] }, diff --git a/openapi/powercontext.yaml b/openapi/powercontext.yaml index f5ece75347..49190dc7f1 100644 --- a/openapi/powercontext.yaml +++ b/openapi/powercontext.yaml @@ -25,6 +25,65 @@ security: - BearerAuth: [] - {} paths: + /v1/atomic-memory/directory/query: + post: + tags: + - atomic-memory + summary: Query compact current Atomic Memory references + description: >- + Returns current metadata and exact revisions without loading Memory text or history. + Any directory or tag change invalidates continuation; restart from the first page on + invalid_cursor or cursor_expired. Requires verified directory migration and writer + composition, and a supported local authorization configuration. Production enablement + is unavailable until the unified migration bundle manages the complete target. + Unavailable codes include artifact_owner_pending, atomic_memory_directory_authorization_unavailable + and atomic_memory_directory_unavailable; do not treat failure as an empty page. + operationId: query_atomic_memory_directory + x-powercontext-scope-mode: current + x-powercontext-access: {action: scope.read, resource: {type: scope, scope-id-from: scope_id}} + responses: + '200': + description: Compact current directory page. + content: + application/json: + schema: + $ref: '#/components/schemas/QueryAtomicMemoryDirectoryResponse' + '400': + description: The cursor is invalid or its query, identity, policy or directory generation changed. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '410': + description: The cursor expired; restart from the first page. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '413': + description: A compact item exceeds the response budget; no partial page is returned. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '422': + $ref: '#/components/responses/InvalidRequest' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/QueryAtomicMemoryDirectoryRequest' /v1/atomic-memory/list: post: tags: @@ -11672,3 +11731,98 @@ components: items: $ref: "#/components/schemas/AccessAuditEvent" next_cursor: {type: string, maxLength: 2048, nullable: true} + QueryAtomicMemoryDirectoryRequest: + type: object + additionalProperties: false + required: + - scope_id + properties: + scope_id: + type: string + minLength: 1 + maxLength: 256 + pattern: .*\S.* + states: + type: array + minItems: 1 + maxItems: 4 + uniqueItems: true + default: [active] + items: + $ref: '#/components/schemas/AtomicMemoryState' + kind: + type: string + minLength: 1 + maxLength: 128 + pattern: .*\S.* + nullable: true + tags: + type: array + maxItems: 16 + default: [] + items: + type: string + minLength: 1 + maxLength: 64 + tag_match: + $ref: '#/components/schemas/TagMatch' + default: all + limit: + type: integer + minimum: 1 + maximum: 100 + default: 50 + cursor: + type: string + minLength: 1 + maxLength: 4096 + nullable: true + AtomicMemoryDirectoryItem: + type: object + additionalProperties: false + required: + - artifact + - kind + - state + - state_version + - merged_into_id + properties: + artifact: + $ref: '#/components/schemas/ArtifactReference' + kind: + type: string + minLength: 1 + maxLength: 128 + pattern: .*\S.* + state: + $ref: '#/components/schemas/AtomicMemoryState' + state_version: + type: integer + minimum: 0 + merged_into_id: + type: string + minLength: 1 + maxLength: 128 + nullable: true + QueryAtomicMemoryDirectoryResponse: + type: object + additionalProperties: false + required: + - scope_id + - items + - next_cursor + properties: + scope_id: + type: string + minLength: 1 + maxLength: 256 + items: + type: array + maxItems: 100 + items: + $ref: '#/components/schemas/AtomicMemoryDirectoryItem' + next_cursor: + type: string + minLength: 1 + maxLength: 4096 + nullable: true diff --git a/src/powercontext/builtin/artifacts/merge.py b/src/powercontext/builtin/artifacts/merge.py index 5fdf8cc13a..4f0d136a07 100644 --- a/src/powercontext/builtin/artifacts/merge.py +++ b/src/powercontext/builtin/artifacts/merge.py @@ -122,7 +122,7 @@ def __init__( sources: SourceRepository | None = None, ) -> None: self.artifacts = artifacts - self.states = ArtifactGovernanceRepository() + self.states = ArtifactGovernanceRepository(atomic_memory_directory=artifacts.atomic_memory_directory) self.adapter = adapter self.family = adapter.family self.errors = errors or ArtifactMergeErrors() diff --git a/src/powercontext/builtin/persistence/artifact_governance.py b/src/powercontext/builtin/persistence/artifact_governance.py index 63659b9627..b8ee0aa482 100644 --- a/src/powercontext/builtin/persistence/artifact_governance.py +++ b/src/powercontext/builtin/persistence/artifact_governance.py @@ -17,6 +17,7 @@ from __future__ import annotations from enum import StrEnum +from typing import TYPE_CHECKING from pydantic import BaseModel, ConfigDict, Field, model_validator from sqlalchemy import select, update @@ -27,6 +28,9 @@ from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE from powercontext.errors import PowerContextError +if TYPE_CHECKING: + from powercontext.builtin.persistence.atomic_memory_directory import AtomicMemoryDirectoryRepository + class InvalidArtifactLifecycleError(PowerContextError, ValueError): """Raised when an explicit governance transition is not valid.""" @@ -67,6 +71,9 @@ def validate_replacement(self) -> ArtifactGovernance: class ArtifactGovernanceRepository: """Read and transition governance state without changing the head Revision.""" + def __init__(self, *, atomic_memory_directory: AtomicMemoryDirectoryRepository | None = None) -> None: + self.atomic_memory_directory = atomic_memory_directory + async def get( self, connection: AsyncConnection, @@ -77,6 +84,8 @@ async def get( *, for_update: bool = False, ) -> ArtifactGovernance: + if for_update and family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) statement = select(ARTIFACT_HEADS_TABLE).where( ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, ARTIFACT_HEADS_TABLE.c.family == family, @@ -116,6 +125,8 @@ async def transition_merge( """Apply a service-validated group transition while retaining unchanged state generations.""" _validate_transition(expected.lifecycle_state, lifecycle_state) + if family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) current = await self.require_summary(connection, scope_id, family, artifact_id, expected) if ( expected.lifecycle_state == lifecycle_state @@ -147,6 +158,8 @@ async def transition_merge( ) if result.rowcount != 1: raise StoredPayloadConflictError("artifact-governance", (scope_id, family, artifact_id)) + if family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.record_mutation(connection, scope_id) return requested async def transition( @@ -160,6 +173,8 @@ async def transition( replacement_artifact_id: str | None, /, ) -> ArtifactGovernance: + if family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) current = await self.get(connection, scope_id, family, artifact_id) if current.merged_into_id is not None: raise InvalidArtifactLifecycleError("frozen merge inputs require whole-group restoration") # noqa: TRY003 @@ -206,6 +221,8 @@ async def transition( raise StoredPayloadConflictError( "artifact-governance", (scope_id, family, artifact_id, expected_generation) ) + if family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.record_mutation(connection, scope_id) return requested diff --git a/src/powercontext/builtin/persistence/artifacts.py b/src/powercontext/builtin/persistence/artifacts.py index 8e16a20afa..0544e597d1 100644 --- a/src/powercontext/builtin/persistence/artifacts.py +++ b/src/powercontext/builtin/persistence/artifacts.py @@ -17,7 +17,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping, Sequence -from typing import Any +from typing import TYPE_CHECKING, Any, cast from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError from sqlalchemy import insert, select, true, tuple_, update @@ -55,6 +55,10 @@ from powercontext.limits import MAX_SCOPE_ID_LENGTH from powercontext.sources import SourceRef +if TYPE_CHECKING: + from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory + from powercontext.builtin.persistence.atomic_memory_directory import AtomicMemoryDirectoryRepository + class RepositoryArtifactDraft(BaseModel): """A family-validated draft accepted by the shared Artifact repository.""" @@ -76,6 +80,7 @@ def __init__( /, *, sources: Any | None = None, + atomic_memory_directory: AtomicMemoryDirectoryRepository | None = None, ) -> None: self._by_family = {artifact_type.family: artifact_type for artifact_type in artifact_types} self._content_types: dict[str, type[BaseModel]] = {} @@ -85,6 +90,8 @@ def __init__( raise TypeError(f"{artifact_type.__name__}.content must be a BaseModel") # noqa: TRY003 self._content_types[family] = content_type self._sources = sources + # Compose only after the unified schema/data capability is ready. + self.atomic_memory_directory = atomic_memory_directory @property def families(self) -> frozenset[str]: @@ -112,6 +119,8 @@ async def create( artifact_type = self._artifact_type(draft.family) self._require_content(draft.family, draft.content) ref = ArtifactRef(family=draft.family, artifact_id=artifact_id, revision=1) + if ref.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) await self._validate_lineage_sources(connection, scope_id, ref, draft.sources) conflict = await self._head_conflict(connection, scope_id, ref, draft) if conflict is not None: @@ -141,6 +150,8 @@ async def create( if conflict is None: raise raise conflict from None + if ref.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.record_publication(connection, scope_id, cast("AtomicMemory", artifact)) return artifact async def create_merge_result( @@ -292,6 +303,8 @@ async def _revise( artifact_id=artifact.artifact_id, revision=artifact.revision + 1, ) + if artifact.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) await self._validate_lineage_sources(connection, scope_id, target, draft.sources) locked = await connection.execute( @@ -355,6 +368,8 @@ async def _revise( ) if advanced.rowcount != 1: raise RevisionConflictError(artifact, revised) + if artifact.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.record_publication(connection, scope_id, cast("AtomicMemory", revised)) return revised async def get( @@ -369,6 +384,8 @@ async def get( """Load one exact revision with ordered direct lineage.""" _require_scope(scope_id) + if for_update and ref.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) statement = select(ARTIFACTS_TABLE).where( ARTIFACTS_TABLE.c.scope_id == scope_id, ARTIFACTS_TABLE.c.family == ref.family, @@ -468,6 +485,8 @@ async def lock_heads( """ _require_scope(scope_id) + if self.atomic_memory_directory is not None and any(ref.family == "atomic-memory" for ref in refs): + await self.atomic_memory_directory.lock_scope(connection, scope_id) for family, artifact_id in sorted({(ref.family, ref.artifact_id) for ref in refs}): locked = await connection.execute( update(ARTIFACT_HEADS_TABLE) @@ -541,6 +560,8 @@ async def copy_exact( _require_scope(target_scope_id) artifact_type = self._artifact_type(source.family) ref = ArtifactRef(family=source.family, artifact_id=target_artifact_id, revision=1) + if ref.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, target_scope_id) copied = await self._insert_revision( connection, target_scope_id, @@ -560,6 +581,10 @@ async def copy_exact( revision=ref.revision, ) ) + if ref.family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.record_publication( + connection, target_scope_id, cast("AtomicMemory", copied) + ) return copied async def _insert_revision( @@ -865,6 +890,8 @@ async def _find_head( *, for_update: bool = False, ) -> int | None: + if for_update and family == "atomic-memory" and self.atomic_memory_directory is not None: + await self.atomic_memory_directory.lock_scope(connection, scope_id) statement = select(ARTIFACT_HEADS_TABLE.c.revision).where( ARTIFACT_HEADS_TABLE.c.scope_id == scope_id, ARTIFACT_HEADS_TABLE.c.family == family, diff --git a/src/powercontext/builtin/persistence/atomic_memory.py b/src/powercontext/builtin/persistence/atomic_memory.py index d871100818..4a8c70d80a 100644 --- a/src/powercontext/builtin/persistence/atomic_memory.py +++ b/src/powercontext/builtin/persistence/atomic_memory.py @@ -14,6 +14,10 @@ """Atomic Memory state adapter over authoritative shared Artifact heads.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + from sqlalchemy import case from powercontext.builtin.artifacts.atomic_memory.errors import AtomicMemoryConflictError, AtomicMemoryRelationError @@ -27,12 +31,15 @@ from powercontext.builtin.persistence.errors import StoredPayloadConflictError from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE +if TYPE_CHECKING: + from powercontext.builtin.persistence.atomic_memory_directory import AtomicMemoryDirectoryRepository + class AtomicMemoryStateRepository: """Translate Atomic vocabulary through the shared governance CAS boundary.""" - def __init__(self) -> None: - self.governance = ArtifactGovernanceRepository() + def __init__(self, *, atomic_memory_directory: AtomicMemoryDirectoryRepository | None = None) -> None: + self.governance = ArtifactGovernanceRepository(atomic_memory_directory=atomic_memory_directory) async def get(self, connection, scope_id, artifact_id, /, *, for_update=False) -> AtomicMemoryState: current = await self.governance.get(connection, scope_id, "atomic-memory", artifact_id, for_update=for_update) diff --git a/src/powercontext/builtin/persistence/atomic_memory_directory.py b/src/powercontext/builtin/persistence/atomic_memory_directory.py new file mode 100644 index 0000000000..86020c6297 --- /dev/null +++ b/src/powercontext/builtin/persistence/atomic_memory_directory.py @@ -0,0 +1,390 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Internal compact directory storage over expanded shared Scope/Artifact tables. + +The caller owns authorization, readiness and a consistent connection. This +repository neither opens transactions nor enables an application operation. +Production schema/readiness and writer composition must be integrated before publication. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from hashlib import sha256 +from typing import Any, Literal, cast + +import rfc8785 +from pydantic import BaseModel, ConfigDict, Field, StrictInt, ValidationError, field_validator +from sqlalchemy import BigInteger, ColumnElement, column, literal_column, select, table, update +from sqlalchemy.engine import RowMapping +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.artifacts import ArtifactRef +from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryContent, AtomicMemoryState +from powercontext.builtin.persistence.atomic_memory import atomic_memory_state_expression +from powercontext.builtin.persistence.codec import load_model +from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE, identity_string +from powercontext.builtin.persistence.tags import tag_predicate +from powercontext.builtin.records import BaseAccessError +from powercontext.builtin.sources import validate_scope_id +from powercontext.builtin.tags import TagFilter + +DIRECTORY_ITEM_BYTES = 4 * 1024 * 1024 +DIRECTORY_ENVELOPE_BYTES = 8192 +DIRECTORY_CURSOR_LENGTH = 4096 +_MAX_GENERATION = (1 << 63) - 1 + +# Expression views only: they do not mutate startup metadata or execute DDL. +# The production schema belongs to the unified frozen migration resources. +_SCOPES = table( + "pc_scopes", + column("scope_id", identity_string(256)), + column("atomic_memory_directory_generation", BigInteger()), +) +_HEADS = table( + "pc_artifact_heads", + column("scope_id", identity_string(256)), + column("family", identity_string(128)), + column("artifact_id", identity_string(128)), + column("revision"), + column("content_kind", identity_string(128)), +) +_KIND = literal_column("pc_artifact_heads.content_kind", type_=identity_string(128)) + + +class AtomicMemoryDirectoryUnavailableError(BaseAccessError): + """Required directory data does not satisfy the declared storage contract.""" + + +class AtomicMemoryDirectoryAuthorizationUnavailableError(BaseAccessError): + """The configured access authorities cannot safely qualify this directory.""" + + +class AtomicMemoryDirectoryItemTooLargeError(BaseAccessError): + """A single compact item cannot fit the item-payload allowance.""" + + +class AtomicMemoryDirectoryQuery(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + scope_id: str + states: tuple[Literal["active", "forgotten", "merged", "retired"], ...] = Field( + default=("active",), min_length=1, max_length=4 + ) + kind: str | None = None + tag_filter: TagFilter | None = None + limit: StrictInt = Field(default=50, ge=1, le=100) + cursor: str | None = Field(default=None, min_length=1, max_length=DIRECTORY_CURSOR_LENGTH) + + @field_validator("scope_id") + @classmethod + def valid_scope(cls, value: str) -> str: + return validate_scope_id(value) + + @field_validator("states") + @classmethod + def distinct_states(cls, values): + if len(set(values)) != len(values): + raise ValueError("states must be distinct") # noqa: TRY003 + return values + + @field_validator("kind") + @classmethod + def valid_kind(cls, value: str | None) -> str | None: + return None if value is None else AtomicMemoryContent(kind=value, text="validation").kind + + +class AtomicMemoryDirectoryItem(AtomicMemoryState): + artifact: ArtifactRef + kind: str = Field(min_length=1, max_length=128) + + @field_validator("kind") + @classmethod + def valid_kind(cls, value: str) -> str: + return AtomicMemoryContent(kind=value, text="validation").kind + + +class AtomicMemoryDirectoryPage(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + scope_id: str + items: tuple[AtomicMemoryDirectoryItem, ...] + next_cursor: str | None = None + + +class AtomicMemoryDirectoryRepository: + """Select page metadata using a caller-supplied, authorized SQL eligibility. + + Eligibility and policy_context must come from the shared authorization + authority. Neither arbitrary SQL nor a string context proves authorization. + An application adapter must enforce that boundary before exposing query(). + """ + + def __init__(self, cursor: SignedCursorCodec, *, available: bool = True) -> None: + self.cursor = cursor + self.available = available + + async def lock_scope(self, connection: AsyncConnection, scope_id: str) -> None: + """Serialize directory-enabled writers before they acquire any head locks. + + A no-op UPDATE also takes SQLite's write lock. It does not invalidate + cursors; only effective publications and mutations advance generation. + """ + + result = await connection.execute( + update(_SCOPES) + .where( + _SCOPES.c.scope_id == scope_id, + _SCOPES.c.atomic_memory_directory_generation >= 0, + _SCOPES.c.atomic_memory_directory_generation <= _MAX_GENERATION, + ) + .values(atomic_memory_directory_generation=_SCOPES.c.atomic_memory_directory_generation) + ) + if result.rowcount != 1: + raise AtomicMemoryDirectoryUnavailableError + + async def query( + self, + connection: AsyncConnection, + query: AtomicMemoryDirectoryQuery, + *, + subject: str, + eligibility: ColumnElement[bool], + policy_context: str, + ) -> AtomicMemoryDirectoryPage: + if not self.available: + raise AtomicMemoryDirectoryUnavailableError + generation = await connection.scalar( + select(_SCOPES.c.atomic_memory_directory_generation).where(_SCOPES.c.scope_id == query.scope_id) + ) + if type(generation) is not int or not 0 <= generation <= _MAX_GENERATION: + raise AtomicMemoryDirectoryUnavailableError + filters = { + "states": sorted(query.states), + "kind": query.kind, + "tags": None if query.tag_filter is None else list(query.tag_filter.keys), + "tag_match": None if query.tag_filter is None else query.tag_filter.match, + "limit": query.limit, + } + bound = { + "endpoint": "atomic_memory_directory", + "version": 1, + "order": "artifact_id_asc_v1", + "context": _digest({"scope_id": query.scope_id, "subject": subject, "policy": policy_context}), + "query": _digest(filters), + "generation": generation, + } + after = self.cursor.after_text(query.cursor, bound) + heads = ARTIFACT_HEADS_TABLE + statement = ( + select( + heads.c.artifact_id, + heads.c.revision, + _KIND.label("kind"), + atomic_memory_state_expression().label("state"), + heads.c.governance_generation.label("state_version"), + heads.c.merged_into_id, + heads.c.replacement_artifact_id, + ) + .where( + heads.c.scope_id == query.scope_id, + heads.c.family == "atomic-memory", + heads.c.artifact_id > after, + atomic_memory_state_expression().in_(query.states), + eligibility, + ) + .order_by(heads.c.artifact_id) + .limit(query.limit + 1) + ) + if query.kind is not None: + statement = statement.where(query.kind == _KIND) + if query.tag_filter is not None: + statement = statement.where( + tag_predicate( + query.scope_id, + "atomic-memory", + heads.c.artifact_id, + "artifact", + heads.c.artifact_id, + query.tag_filter, + ) + ) + rows = (await connection.execute(statement)).mappings().all() + items: list[AtomicMemoryDirectoryItem] = [] + # Account for the array brackets as well as each comma-separated item. + item_bytes = 2 + for row in rows[: query.limit]: + item = _directory_item(row) + encoded_size = len(item.model_dump_json().encode("utf-8")) + if encoded_size + 2 > DIRECTORY_ITEM_BYTES: + raise AtomicMemoryDirectoryItemTooLargeError + next_size = item_bytes + encoded_size + bool(items) + if next_size > DIRECTORY_ITEM_BYTES: + break + items.append(item) + item_bytes = next_size + has_more = len(rows) > len(items) + next_cursor = self.cursor.encode(bound, items[-1].artifact.artifact_id) if has_more else None + if next_cursor is not None and len(next_cursor) > DIRECTORY_CURSOR_LENGTH: + raise AtomicMemoryDirectoryUnavailableError + page = AtomicMemoryDirectoryPage(scope_id=query.scope_id, items=tuple(items), next_cursor=next_cursor) + envelope = page.model_copy(update={"items": ()}) + if len(envelope.model_dump_json().encode("utf-8")) - 2 > DIRECTORY_ENVELOPE_BYTES: + raise AtomicMemoryDirectoryUnavailableError + return page + + async def record_publication(self, connection: AsyncConnection, scope_id: str, artifact: AtomicMemory) -> None: + """Maintain metadata after an effective publish, inside its business transaction.""" + + result = await connection.execute( + update(_HEADS) + .where( + _HEADS.c.scope_id == scope_id, + _HEADS.c.family == artifact.family, + _HEADS.c.artifact_id == artifact.artifact_id, + _HEADS.c.revision == artifact.revision, + ) + .values(content_kind=artifact.content.kind) + ) + if result.rowcount != 1: + raise AtomicMemoryDirectoryUnavailableError + await self.record_mutation(connection, scope_id) + + async def maintain_batch( # noqa: C901 + self, + connection: AsyncConnection, + scope_id: str, + *, + after: str = "", + batch_size: int = 200, + repair: bool = False, + ) -> tuple[str | None, int]: + """Validate or repair current metadata for an offline domain-task executor. + + Return (next position, changed heads). The caller owns maintenance locks, + backup, schema readiness, transactions and the full-Scope iteration. + Verification never writes. Repair reads exact current content only and + advances generation only when this batch actually changes metadata. + """ + + validate_scope_id(scope_id) + if type(batch_size) is not int or not 1 <= batch_size <= 200 or type(repair) is not bool: + raise ValueError("invalid directory maintenance batch") # noqa: TRY003 + if after: + ArtifactRef(family="atomic-memory", artifact_id=after, revision=1) + if repair: + await self.lock_scope(connection, scope_id) + generation = await connection.scalar( + select(_SCOPES.c.atomic_memory_directory_generation).where(_SCOPES.c.scope_id == scope_id) + ) + if type(generation) is not int or not 0 <= generation <= _MAX_GENERATION: + raise AtomicMemoryDirectoryUnavailableError + heads, artifacts = ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE + rows = ( + ( + await connection.execute( + select( + heads.c.artifact_id, + heads.c.revision, + _KIND.label("kind"), + atomic_memory_state_expression().label("state"), + heads.c.governance_generation.label("state_version"), + heads.c.merged_into_id, + heads.c.replacement_artifact_id, + artifacts.c.content, + ) + .select_from( + heads.outerjoin( + artifacts, + (artifacts.c.scope_id == heads.c.scope_id) + & (artifacts.c.family == heads.c.family) + & (artifacts.c.artifact_id == heads.c.artifact_id) + & (artifacts.c.revision == heads.c.revision), + ) + ) + .where( + heads.c.scope_id == scope_id, + heads.c.family == "atomic-memory", + heads.c.artifact_id > after, + ) + .order_by(heads.c.artifact_id) + .limit(batch_size + 1) + ) + ) + .mappings() + .all() + ) + changes = [] + # Validate the entire batch before changing any head. An outer join + # ensures a missing exact body cannot disappear from verification. + for row in rows[:batch_size]: + content = load_model(AtomicMemoryContent, row["content"], kind="artifact", name="atomic-memory") + _directory_item({**row, "kind": content.kind}) + if row["kind"] != content.kind: + if not repair: + raise AtomicMemoryDirectoryUnavailableError + changes.append((row["artifact_id"], row["revision"], content.kind)) + for identity, revision, kind in changes: + result = await connection.execute( + update(_HEADS) + .where( + _HEADS.c.scope_id == scope_id, + _HEADS.c.family == "atomic-memory", + _HEADS.c.artifact_id == identity, + _HEADS.c.revision == revision, + ) + .values(content_kind=kind) + ) + if result.rowcount != 1: + raise AtomicMemoryDirectoryUnavailableError + if changes: + await self.record_mutation(connection, scope_id) + next_after = str(rows[batch_size - 1]["artifact_id"]) if len(rows) > batch_size else None + return next_after, len(changes) + + async def record_mutation(self, connection: AsyncConnection, scope_id: str) -> None: + """Invalidate cursors after an effective state/tag/repair mutation; never commit.""" + + result = await connection.execute( + update(_SCOPES) + .where( + _SCOPES.c.scope_id == scope_id, + _SCOPES.c.atomic_memory_directory_generation >= 0, + _SCOPES.c.atomic_memory_directory_generation < _MAX_GENERATION, + ) + .values(atomic_memory_directory_generation=_SCOPES.c.atomic_memory_directory_generation + 1) + ) + if result.rowcount != 1: + raise AtomicMemoryDirectoryUnavailableError + + +def _digest(value: Mapping[str, Any]) -> str: + return sha256(rfc8785.dumps(cast(Any, value))).hexdigest() + + +def _directory_item(row: RowMapping | Mapping[str, Any]) -> AtomicMemoryDirectoryItem: + if row["replacement_artifact_id"] is not None: + raise AtomicMemoryDirectoryUnavailableError + try: + return AtomicMemoryDirectoryItem( + artifact=ArtifactRef(family="atomic-memory", artifact_id=row["artifact_id"], revision=row["revision"]), + kind=row["kind"], + state=row["state"], + state_version=row["state_version"], + merged_into_id=row["merged_into_id"], + ) + except (ValidationError, ValueError) as error: + raise AtomicMemoryDirectoryUnavailableError from error diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py index c38bc2671e..a840aa15ea 100644 --- a/src/powercontext/builtin/persistence/migrations/bundle.py +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -18,6 +18,7 @@ import hashlib import json +import runpy import threading from pathlib import Path from tempfile import TemporaryDirectory @@ -40,7 +41,7 @@ class MigrationBundle: partially applied DDL cannot be mistaken for a supported baseline. """ - def __init__(self, directory: Path) -> None: + def __init__(self, directory: Path, *, target: str | None = None) -> None: # noqa: C901 self.directory = directory.resolve() # Execute exactly the reviewed bytes, even if the installed resources # change during a long backup. Alembic reloads scripts on every command. @@ -52,11 +53,22 @@ def __init__(self, directory: Path) -> None: self._snapshot = TemporaryDirectory(prefix="pc-migration-bundle-") self.snapshot_directory = Path(self._snapshot.name).resolve() for name, content in resources.items(): - target = self.snapshot_directory / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(content) + resource_target = self.snapshot_directory / name + resource_target.parent.mkdir(parents=True, exist_ok=True) + resource_target.write_bytes(content) + manifest = json.loads(resources["manifest.json"]) + selected = manifest.get("targets", {}).get(target) if target is not None else None + if target is not None and selected is None: + raise MigrationError("unsupported_target", "The release does not register this migration target.") + head = selected["revision"] if selected is not None else manifest.get("default_revision", "heads") self.script = ScriptDirectory(str(self.snapshot_directory)) - revisions = list(reversed(list(self.script.walk_revisions()))) + known_revisions = {item.revision for item in self.script.walk_revisions()} + if ( + set(manifest["sqlite_fingerprints"]) != known_revisions + or not set(manifest["sqlite_baselines"].values()) <= known_revisions + ): + raise MigrationError("invalid_bundle", "Schema inventory must cover the registered history and baselines.") + revisions = list(reversed(list(self.script.walk_revisions(head=head)))) previous = None for item in revisions: if item.down_revision != previous or item.dependencies or len(item.revision) > 32: @@ -66,8 +78,7 @@ def __init__(self, directory: Path) -> None: raise MigrationError("invalid_bundle", "Migration history is empty.") self.revisions = tuple(item.revision for item in revisions) self.head = self.revisions[-1] - manifest = json.loads(resources["manifest.json"]) - self.affected_objects = tuple(manifest.get("affected_objects", ())) + self.affected_objects = tuple((selected or manifest).get("affected_objects", ())) self.retained_objects = tuple(manifest.get("retained_objects", ())) self.retained_fingerprints = manifest.get("retained_fingerprints", {}) self.retained_revisions = manifest.get("retained_revisions", {}) @@ -75,10 +86,17 @@ def __init__(self, directory: Path) -> None: self.retained_revisions.values() ) <= set(self.revisions): raise MigrationError("invalid_bundle", "Retained objects must declare their introducing revision.") - self.fingerprints: dict[str, list[str]] = manifest["sqlite_fingerprints"] - self.baselines: dict[str, str] = manifest["sqlite_baselines"] - if set(self.fingerprints) != set(self.revisions) or not set(self.baselines.values()) <= set(self.revisions): + if not set(self.revisions) <= set(manifest["sqlite_fingerprints"]): raise MigrationError("invalid_bundle", "Schema inventory does not cover the revision chain.") + self.fingerprints: dict[str, list[str]] = { + revision: manifest["sqlite_fingerprints"][revision] for revision in self.revisions + } + self.baselines: dict[str, str] = { + fingerprint: revision + for fingerprint, revision in manifest["sqlite_baselines"].items() + if revision in self.revisions and (selected is None or revision in selected["baseline_revisions"]) + } + self.minimum_source_revision = None if selected is None else selected["minimum_source_revision"] if any(fingerprint not in self.fingerprints[revision] for fingerprint, revision in self.baselines.items()): raise MigrationError("invalid_bundle", "A baseline does not match its declared revision.") self.resources = {name: hashlib.sha256(content).hexdigest() for name, content in resources.items()} @@ -90,7 +108,32 @@ def __init__(self, directory: Path) -> None: Path(item.path).read_bytes() ).hexdigest() self.checksums[item.revision] = digest(inputs) - self.checksum = digest(self.resources) + self.checksum = digest({"resources": self.resources, "target_revision": self.head, "target": selected}) + self.data_tasks = tuple(task for task in manifest.get("data_tasks", ()) if task["revision"] in self.revisions) + for task in self.data_tasks: + if task["resource"] not in manifest["revision_resources"][task["revision"]]: + raise MigrationError("invalid_bundle", "A data task must be bound to its revision resources.") + + def task_functions(self, task: dict[str, str]) -> dict[str, Any]: + """Load only the immutable resource snapshot used by the reviewed plan.""" + + self.assert_unchanged() + return runpy.run_path(str(self.snapshot_directory / task["resource"])) + + def pending_data_tasks(self, connection: Connection, revision: str | None) -> tuple[str, ...]: + """Inspect current data, independently of the schema revision or old completion evidence.""" + + completed = set(self.revisions) - set(self.pending(revision)) + pending = [] + for task in self.data_tasks: + try: + if task["revision"] not in completed or not self.task_functions(task)["verify"](connection): + pending.append(task["name"]) + except ValueError as error: + raise MigrationError( + "verification_failed", "Registered data does not satisfy the frozen domain format." + ) from error + return tuple(pending) def assert_unchanged(self) -> None: resources = { diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 34bbd9156b..4cc4c492ef 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -17,6 +17,8 @@ from __future__ import annotations import os +import sqlite3 +from contextlib import closing from pathlib import Path from typing import TYPE_CHECKING, Literal @@ -25,6 +27,7 @@ from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.persistence.sqlite.profile import _readonly_sqlite_url from .bundle import MigrationBundle from .connections import MaintenanceConnections @@ -36,23 +39,100 @@ from powercontext.server.settings import ServerSettings -def production_bundle() -> MigrationBundle: +def production_bundle(*, target: Literal["atomic-memory-directory"] | None = None) -> MigrationBundle: """Load installed resources; readiness is scoped to this acceptance bundle.""" - return MigrationBundle(Path(__file__).parent / "resources") + return MigrationBundle(Path(__file__).parent / "resources", target=target) + + +def _directory_runner(database: SQLiteConfig) -> tuple[SQLiteMigrationRunner, str | None] | None: + if not database.is_persistent: + return None + url = make_url(database.url) + if not url.database or url.host or url.username or url.password or url.port: + return None + readonly_url = _readonly_sqlite_url(database.url).set(drivername="sqlite") + arguments, _ = readonly_url.get_dialect()().create_connect_args(readonly_url) + try: + inspection = sqlite3.connect(str(arguments[0]), uri=True, timeout=database.busy_timeout_ms / 1000) + except sqlite3.OperationalError as error: + if getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_CANTOPEN: + return None + raise + with closing(inspection) as connection: + path = Path(next(row[2] for row in connection.execute("PRAGMA database_list") if row[1] == "main")).resolve() + exists = connection.execute("SELECT 1 FROM sqlite_schema WHERE name='pc_schema_revision'").fetchone() + if exists is None and not path.with_name(path.name + ".pc-migration-state").exists(): + return None + revisions = ( + [] if exists is None else connection.execute("SELECT version_num FROM pc_schema_revision").fetchall() + ) + if len(revisions) > 1: + raise MigrationError("incompatible_schema", "Multiple schema revisions are not supported.") + revision = None if not revisions else str(revisions[0][0]) + runner = SQLiteMigrationRunner( + path, + production_bundle(target="atomic-memory-directory"), + configuration=database.model_dump(exclude={"url", "echo"}), + ) + return runner, revision + + +def assert_directory_maintenance_complete(database: SQLiteConfig) -> None: + """Check only maintenance evidence; do not scan domain data for each component open.""" + + inspected = _directory_runner(database) + if inspected is not None: + inspected[0].assert_business_startup_allowed() + + +def directory_readiness(database: SQLiteConfig) -> bool | None: + """Inspect the optional directory without initialization or repair. + + False retains writer hooks but disables querying. None is an unmanaged + directory. The shared executor owns data and maintenance classification. + """ + + inspected = _directory_runner(database) + if inspected is None: + return None + runner, revision = inspected + runner.assert_business_startup_allowed() + if revision is None or revision in {"p0001", "p0002", "p0003"}: + return None + if revision not in {"p0004", "p0005"}: + raise MigrationError("incompatible_schema", "The schema revision is not supported by this program.") + try: + plan = runner.plan() + except MigrationError as error: + if error.code == "verification_failed": + return False + raise + if plan.state == "recovery_required": + raise MigrationError("recovery_required", "Complete explicit directory maintenance before business startup.") + return plan.state == "ready" and not make_url(database.url).query if revision == "p0005" else None MigrationRunner = SQLiteMigrationRunner | MySQLMigrationRunner -def deployment_runner( +def deployment_runner( # noqa: C901 - one authority for target/backend/coordination eligibility settings: ServerSettings, *, evidence_dir: Path | None = None, lock_coordination: Literal["single-host"] | None = None, acknowledge_previous_run: str | None = None, + target: Literal["atomic-memory-directory"] | None = None, ) -> MigrationRunner: """Use the real configured identity without opening a business profile.""" database = settings.database + if target is not None and evidence_dir is not None: + raise MigrationError( + "unsupported_option", "The directory target uses the database's default evidence directory." + ) + if target is not None and not isinstance(database, SQLiteConfig): + raise MigrationError("unsupported_target", "The directory target is registered only for persistent SQLite.") + if target is not None and settings.inference.embedding_model is not None: + raise MigrationError("unsupported_target", "The registered directory target has no embedding extension.") if acknowledge_previous_run is not None and not isinstance(database, OceanBaseConfig): raise MigrationError("unsupported_option", "--acknowledge-previous-run is supported only for OceanBase.") if isinstance(database, (SeekDBConfig, OceanBaseConfig)): @@ -62,8 +142,8 @@ def deployment_runner( "evidence_required", "OceanBase maintenance requires --evidence-dir on one fixed host, shared by all migration Jobs.", ) - target = database.path.expanduser().resolve() - evidence_dir = target.with_name(target.name + ".pc-migration-state") + engine_directory = database.path.expanduser().resolve() + evidence_dir = engine_directory.with_name(engine_directory.name + ".pc-migration-state") if isinstance(database, OceanBaseConfig) and lock_coordination != "single-host": raise MigrationError( "coordination_required", @@ -88,10 +168,10 @@ def deployment_runner( "unsupported_target", "Use a regular SQLite file URL without credentials, host, port or URI options." ) # Match SQLite's URL normalization before the runner expands standalone Path inputs. - target = Path(os.path.abspath(url.database)) + database_path = Path(os.path.abspath(url.database)) return SQLiteMigrationRunner( - target, - production_bundle(), + database_path, + production_bundle(target=target), configuration=database.model_dump(exclude={"url", "echo"}), evidence_directory=evidence_dir, ) diff --git a/src/powercontext/builtin/persistence/migrations/models.py b/src/powercontext/builtin/persistence/migrations/models.py index abbbd7af57..d0ccfc965f 100644 --- a/src/powercontext/builtin/persistence/migrations/models.py +++ b/src/powercontext/builtin/persistence/migrations/models.py @@ -48,6 +48,7 @@ class MigrationPlan(BaseModel): configuration_digest: str state: Literal["uninitialized", "migration_required", "ready", "recovery_required"] revisions: tuple[str, ...] + data_tasks: tuple[str, ...] = () adopt_baseline: bool = False backup_policy: Literal["auto", "manual", "skip"] = "auto" backup_available: bool = True diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index abfccff3a6..be05598e9f 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -148,6 +148,8 @@ def __init__( configuration: Mapping[str, Any] | None = None, backup_factory: BackupFactory | None = None, ) -> None: + if bundle.data_tasks: + raise MigrationError("unsupported_target", "This backend has no registered data-task execution adapter.") self.connections = connections self.bundle = bundle self.schema = MySQLSchema(bundle) diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index 65c798ea8b..2ec2a998f8 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -4,11 +4,19 @@ "schemas/pre_dream.json" ], "p0002": [], - "p0003": [] + "p0003": [], + "p0004": [ + "schemas/atomic_directory_sqlite_v1.json" + ], + "p0005": [ + "schemas/atomic_directory_sqlite_v1.json", + "tasks/atomic_directory_v1.py" + ] }, "sqlite_baselines": { "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", - "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003" + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003", + "11e8770068a5fe32dfb25681564dbbe586ba0fd0238ed01adf1b09dde8a49f22": "p0004" }, "sqlite_fingerprints": { "p0001": [ @@ -20,6 +28,12 @@ "p0003": [ "9958fa0413ed3c16442df27dc48057d438e2cf3f7e4312c5a914cccd9d3acaab", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d" + ], + "p0004": [ + "11e8770068a5fe32dfb25681564dbbe586ba0fd0238ed01adf1b09dde8a49f22" + ], + "p0005": [ + "535851738dc14ba0e1d4ebb29a0cfc13e39f31fc70fb0ed7a6a479a808cafdd5" ] }, "affected_objects": [ @@ -38,5 +52,85 @@ }, "retained_revisions": { "pc_retained_p0002_artifact_tags": "p0003" - } + }, + "default_revision": "p0003", + "targets": { + "atomic-memory-directory": { + "revision": "p0005", + "affected_objects": [ + "pc_scopes", + "pc_artifact_heads", + "pc_access_audit", + "pc_access_idempotency", + "pc_access_owners", + "pc_access_relationship_heads", + "pc_access_relationships", + "pc_agent_skill_targets", + "pc_artifact_candidate_heads", + "pc_artifact_candidate_versions", + "pc_artifact_fts", + "pc_artifact_lineage_artifacts", + "pc_artifact_lineage_sources", + "pc_artifact_processing_auto_wave_targets", + "pc_artifact_processing_binding_states", + "pc_artifact_processing_intents", + "pc_artifact_processing_leases", + "pc_artifact_processing_migration_receipts", + "pc_artifact_processing_pending", + "pc_artifact_processing_schema", + "pc_artifact_processing_sequences", + "pc_artifact_publications", + "pc_artifact_tags", + "pc_artifacts", + "pc_atomic_memory_current", + "pc_atomic_memory_current_fts", + "pc_connector_checkpoints", + "pc_decision_observations", + "pc_dream_runs", + "pc_external_skill_registrations", + "pc_memory_entry_heads", + "pc_memory_entry_versions", + "pc_memory_source_windows", + "pc_model_usage_daily", + "pc_profile_policies", + "pc_recall_effort_daily", + "pc_recall_token_daily", + "pc_receipt_migration_review", + "pc_recurrence_match", + "pc_recurrence_observation", + "pc_scope_bindings", + "pc_scope_context_references", + "pc_scope_creation_requests", + "pc_scope_external_references", + "pc_scope_settings", + "pc_server_identity", + "pc_skill_packages", + "pc_skill_publications", + "pc_source_cursors", + "pc_source_definition_manifests", + "pc_source_journal_heads", + "pc_sources", + "pc_topic_memory_active_chunks", + "pc_topic_memory_active_topics", + "pc_topic_memory_chunk_fts", + "pc_topic_memory_fts_index", + "pc_topic_memory_processing_targets", + "pc_topic_memory_retrieval_shape", + "pc_topic_memory_revision_publications", + "pc_topic_memory_topic_fts", + "pc_topic_memory_work_budgets" + ], + "baseline_revisions": [ + "p0004" + ], + "minimum_source_revision": "p0004" + } + }, + "data_tasks": [ + { + "name": "atomic-memory-directory-v1", + "revision": "p0005", + "resource": "tasks/atomic_directory_v1.py" + } + ] } diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_directory_sqlite_v1.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_directory_sqlite_v1.json new file mode 100644 index 0000000000..f9279ab4e9 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/atomic_directory_sqlite_v1.json @@ -0,0 +1,154 @@ +{ + "provenance": { + "commit": "b5bcfe24bcce9c574ee85175d623bb9963970a41", + "configuration": "persistent SQLite, default Server/Runtime, no embedding extension", + "scope": "current business structural baseline; legacy data conversions remain outside this target" + }, + "sqlite_init": [ + "CREATE TABLE pc_access_audit (\n\tcursor INTEGER NOT NULL, \n\tevent_id VARCHAR(64) NOT NULL, \n\toccurred_at VARCHAR(32) NOT NULL, \n\trequest_id VARCHAR(128), \n\ttransport VARCHAR(16) NOT NULL, \n\toperation VARCHAR(128) NOT NULL, \n\tprincipal_type VARCHAR(16) NOT NULL, \n\tprincipal_id VARCHAR(255) NOT NULL, \n\tprincipal_description TEXT, \n\tactor_type VARCHAR(16), \n\tactor_id VARCHAR(255), \n\tactor_description TEXT, \n\taction VARCHAR(64) NOT NULL, \n\tresource_type VARCHAR(16) NOT NULL, \n\tdeployment_id VARCHAR(128), \n\tscope_id VARCHAR(256), \n\tfamily VARCHAR(128), \n\tartifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\tallowed BOOLEAN NOT NULL, \n\treason_code VARCHAR(64) NOT NULL, \n\tpolicy_revision VARCHAR(64), \n\tmatched_subject_type VARCHAR(16), \n\tmatched_subject_id VARCHAR(255), \n\tmatched_subject_description TEXT, \n\tbinding_id VARCHAR(64), \n\ttarget_type VARCHAR(16), \n\ttarget_id VARCHAR(255), \n\ttarget_description TEXT, \n\trole VARCHAR(32), \n\texpected_version INTEGER, \n\tresult_version INTEGER, \n\tPRIMARY KEY (cursor), \n\tUNIQUE (event_id)\n)", + "CREATE TABLE pc_access_idempotency (\n\tactor_id VARCHAR(255) NOT NULL, \n\tidempotency_key_hash VARCHAR(64) NOT NULL, \n\toperation VARCHAR(64) NOT NULL, \n\tpayload_hash VARCHAR(64) NOT NULL, \n\tresult_binding_id VARCHAR(64), \n\tsecondary_binding_id VARCHAR(64), \n\tPRIMARY KEY (actor_id, idempotency_key_hash), \n\tCONSTRAINT uq_pc_access_idempotency_actor_key UNIQUE (actor_id, idempotency_key_hash)\n)", + "CREATE TABLE pc_access_owners (\n\towner_kind VARCHAR(16) NOT NULL, \n\tobject_key_hash VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128), \n\tcandidate_id VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\towner_type VARCHAR(16) NOT NULL, \n\towner_id VARCHAR(255) NOT NULL, \n\towner_description TEXT, \n\testablished_at VARCHAR(32), \n\tpolicy_revision VARCHAR(64), \n\tidempotency_key VARCHAR(255) NOT NULL, \n\tPRIMARY KEY (owner_kind, object_key_hash), \n\tCONSTRAINT ck_pc_access_owners_kind CHECK ((owner_kind = 'artifact' AND artifact_id IS NOT NULL AND candidate_id IS NULL AND target_artifact_id IS NULL AND established_at IS NOT NULL AND policy_revision IS NOT NULL) OR (owner_kind = 'candidate' AND candidate_id IS NOT NULL AND artifact_id IS NULL AND selector_type IS NULL AND selector_entry_id IS NULL AND established_at IS NULL AND policy_revision IS NULL))\n)", + "CREATE TABLE pc_access_relationship_heads (\n\tname VARCHAR(32) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tPRIMARY KEY (name), \n\tCONSTRAINT ck_pc_access_relationship_heads_revision_nonnegative CHECK (revision >= 0)\n)", + "CREATE TABLE pc_access_relationships (\n\tbinding_id VARCHAR(64) NOT NULL, \n\tsubject_type VARCHAR(16) NOT NULL, \n\tsubject_id VARCHAR(255) NOT NULL, \n\tsubject_description TEXT, \n\tresource_key_hash VARCHAR(64) NOT NULL, \n\tresource_type VARCHAR(16) NOT NULL, \n\tdeployment_id VARCHAR(128), \n\tscope_id VARCHAR(256), \n\tfamily VARCHAR(128), \n\tartifact_id VARCHAR(128), \n\tselector_type VARCHAR(32), \n\tselector_entry_id VARCHAR(128), \n\trole VARCHAR(32) NOT NULL, \n\tsingleton_key VARCHAR(64), \n\tgranted_by_type VARCHAR(16) NOT NULL, \n\tgranted_by_id VARCHAR(255) NOT NULL, \n\tgranted_by_description TEXT, \n\treason TEXT, \n\tcreated_at VARCHAR(32) NOT NULL, \n\texpires_at VARCHAR(32), \n\tstate VARCHAR(16) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tpolicy_revision VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(255) NOT NULL, \n\trevoked_at VARCHAR(32), \n\trevoked_by_type VARCHAR(16), \n\trevoked_by_id VARCHAR(255), \n\trevoked_by_description TEXT, \n\tPRIMARY KEY (binding_id), \n\tCONSTRAINT ck_pc_access_relationships_version_positive CHECK (version > 0), \n\tUNIQUE (singleton_key)\n)", + "CREATE TABLE pc_agent_skill_targets (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_id VARCHAR(64) NOT NULL, \n\tdisplay_name VARCHAR(128) NOT NULL, \n\tagent_kind VARCHAR(32) NOT NULL, \n\tinstallation_scope VARCHAR(16) NOT NULL, \n\tdelivery_mode VARCHAR(16) NOT NULL, \n\tinstallation_id VARCHAR(128), \n\tstate VARCHAR(16) NOT NULL, \n\tenrollment_token_digest VARCHAR(64), \n\tenrollment_expires_at DATETIME, \n\tcredential_subject VARCHAR(128), \n\tcredential_verifier VARCHAR(64), \n\treceiver_version VARCHAR(64), \n\tenvironment_fingerprint VARCHAR(64), \n\tmachine_hostname VARCHAR(255), \n\tworkspace_name VARCHAR(128), \n\tlast_seen_at DATETIME, \n\tgeneration BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, target_id), \n\tCONSTRAINT uq_pc_agent_skill_targets_installation UNIQUE (scope_id, agent_kind, installation_scope, installation_id), \n\tCONSTRAINT uq_pc_agent_skill_targets_enrollment_token UNIQUE (enrollment_token_digest), \n\tCONSTRAINT uq_pc_agent_skill_targets_credential_subject UNIQUE (credential_subject), \n\tCONSTRAINT uq_pc_agent_skill_targets_credential_verifier UNIQUE (credential_verifier), \n\tCONSTRAINT ck_pc_agent_skill_targets_agent_kind CHECK (agent_kind IN ('codex', 'claude_code')), \n\tCONSTRAINT ck_pc_agent_skill_targets_installation_scope CHECK (installation_scope IN ('project')), \n\tCONSTRAINT ck_pc_agent_skill_targets_delivery_mode CHECK (delivery_mode = 'agent_pull'), \n\tCONSTRAINT ck_pc_agent_skill_targets_state CHECK (state IN ('pending', 'active', 'revoked')), \n\tCONSTRAINT ck_pc_agent_skill_targets_state_payload CHECK ((state = 'pending' AND enrollment_token_digest IS NOT NULL AND enrollment_expires_at IS NOT NULL AND installation_id IS NULL AND credential_subject IS NULL AND credential_verifier IS NULL) OR (state = 'active' AND enrollment_token_digest IS NULL AND enrollment_expires_at IS NULL AND installation_id IS NOT NULL AND credential_subject IS NOT NULL AND credential_verifier IS NOT NULL) OR (state = 'revoked' AND enrollment_token_digest IS NULL AND enrollment_expires_at IS NULL AND credential_verifier IS NULL)), \n\tCONSTRAINT ck_pc_agent_skill_targets_generation_nonnegative CHECK (generation >= 0)\n)", + "CREATE TABLE pc_artifact_candidate_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\tresult_family VARCHAR(128), \n\tresult_artifact_id VARCHAR(128), \n\tresult_revision INTEGER, \n\tdecision_reason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id), \n\tFOREIGN KEY(scope_id, candidate_id, version) REFERENCES pc_artifact_candidate_versions (scope_id, candidate_id, version) ON DELETE RESTRICT, \n\tFOREIGN KEY(scope_id, result_family, result_artifact_id, result_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_heads_status CHECK (status IN ('pending', 'approved', 'rejected')), \n\tCONSTRAINT ck_pc_artifact_candidate_heads_terminal_result CHECK ((status = 'approved' AND result_family IS NOT NULL AND result_artifact_id IS NOT NULL AND result_revision > 0 AND decision_reason IS NULL) OR (status = 'rejected' AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NOT NULL) OR (status = 'pending' AND result_family IS NULL AND result_artifact_id IS NULL AND result_revision IS NULL AND decision_reason IS NULL))\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE VIRTUAL TABLE pc_artifact_fts USING fts5(\n scope_id UNINDEXED,\n family UNINDEXED,\n artifact_id UNINDEXED,\n revision UNINDEXED,\n searchable_text,\n tokenize='unicode61'\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tmerged_into_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated'), \n\tCONSTRAINT ck_pc_artifact_heads_merge_target CHECK (merged_into_id IS NULL OR (lifecycle_state = 'deprecated' AND merged_into_id <> artifact_id AND replacement_artifact_id IS NULL))\n)", + "CREATE TABLE pc_artifact_lineage_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tupstream_family VARCHAR(128) NOT NULL, \n\tupstream_artifact_id VARCHAR(128) NOT NULL, \n\tupstream_revision INTEGER NOT NULL, \n\tis_merge_input BOOLEAN DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision, ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, upstream_family, upstream_artifact_id, upstream_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT\n)", + "CREATE TABLE pc_artifact_lineage_sources (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision, ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, source_type, source_id) REFERENCES pc_sources (scope_id, source_type, source_id) ON DELETE RESTRICT\n)", + "CREATE TABLE pc_artifact_processing_auto_wave_targets (\n\twave_id VARCHAR(36) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tcompleted BOOLEAN DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (wave_id, binding_name, scope_id), \n\tCONSTRAINT ck_pc_artifact_processing_auto_wave_target_source_positive CHECK (source_through >= 1)\n)", + "CREATE TABLE pc_artifact_processing_binding_states (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tlast_auto_wave_completed_at DATETIME, \n\tlast_schedule_checkpoint_at DATETIME, \n\tscan_generation BIGINT DEFAULT '0' NOT NULL, \n\tscan_in_progress BOOLEAN DEFAULT '0' NOT NULL, \n\tscan_upper_pending_sequence BIGINT, \n\tPRIMARY KEY (binding_name), \n\tCONSTRAINT ck_pc_processing_scan_generation CHECK (scan_generation >= 0)\n)", + "CREATE TABLE pc_artifact_processing_intents (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tpending_sequence BIGINT NOT NULL, \n\tdirty_generation BIGINT DEFAULT '0' NOT NULL, \n\tclean_generation BIGINT DEFAULT '0' NOT NULL, \n\trequested_generation BIGINT DEFAULT '0' NOT NULL, \n\thandled_generation BIGINT DEFAULT '0' NOT NULL, \n\tlast_auto_scan_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_processing_intent_sequence CHECK (pending_sequence > 0), \n\tCONSTRAINT ck_pc_processing_intent_dirty CHECK (clean_generation >= 0 AND clean_generation <= dirty_generation), \n\tCONSTRAINT ck_pc_processing_intent_request CHECK (handled_generation >= 0 AND handled_generation <= requested_generation), \n\tCONSTRAINT ck_pc_processing_intent_scan CHECK (last_auto_scan_generation >= 0), \n\tUNIQUE (pending_sequence)\n)", + "CREATE TABLE pc_artifact_processing_leases (\n\tsupervisor_group VARCHAR(64) NOT NULL, \n\tholder_id VARCHAR(36) NOT NULL, \n\tsupervisor_generation BIGINT NOT NULL, \n\tlease_expires_at DATETIME, \n\tPRIMARY KEY (supervisor_group), \n\tCONSTRAINT ck_pc_artifact_processing_leases_generation_positive CHECK (supervisor_generation > 0)\n)", + "CREATE TABLE pc_artifact_processing_migration_receipts (\n\tmigration_id VARCHAR(64) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_snapshot BLOB NOT NULL, \n\trequested_generation BIGINT NOT NULL, \n\tdirty_generation BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tPRIMARY KEY (migration_id, binding_name, scope_id)\n)", + "CREATE TABLE pc_artifact_processing_pending (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tflush_generation BIGINT DEFAULT '0' NOT NULL, \n\thandled_flush_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_artifact_processing_pending_source_positive CHECK (source_through >= 1), \n\tCONSTRAINT ck_pc_artifact_processing_pending_flush_nonnegative CHECK (flush_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_processing_pending_handled_nonnegative CHECK (handled_flush_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_processing_pending_handled_not_ahead CHECK (handled_flush_generation <= flush_generation)\n)", + "CREATE TABLE pc_artifact_processing_schema (\n\tsingleton INTEGER NOT NULL, \n\tschema_version INTEGER NOT NULL, \n\tsource_version INTEGER DEFAULT '1490' NOT NULL, \n\tphase VARCHAR(32) NOT NULL, \n\tmigration_id VARCHAR(64) NOT NULL, \n\tconfig_manifest TEXT NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_processing_schema_singleton CHECK (singleton = 1)\n)", + "CREATE TABLE pc_artifact_processing_sequences (\n\tsingleton INTEGER NOT NULL, \n\tsequence BIGINT NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_processing_sequence CHECK (singleton = 1 AND sequence >= 0)\n)", + "CREATE TABLE pc_artifact_publications (\n\ttarget_scope_id VARCHAR(256) NOT NULL, \n\ttarget_family VARCHAR(128) NOT NULL, \n\ttarget_artifact_id VARCHAR(128) NOT NULL, \n\ttarget_revision INTEGER NOT NULL, \n\tsource_scope_id VARCHAR(256) NOT NULL, \n\tsource_family VARCHAR(128) NOT NULL, \n\tsource_artifact_id VARCHAR(128) NOT NULL, \n\tsource_revision INTEGER NOT NULL, \n\tcontent_digest VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (target_scope_id, target_family, target_artifact_id, target_revision), \n\tFOREIGN KEY(target_scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tFOREIGN KEY(source_scope_id, source_family, source_artifact_id, source_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT uq_pc_artifact_publications_request UNIQUE (target_scope_id, idempotency_key)\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_atomic_memory_current (\n\tscope_id VARCHAR(256) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision BIGINT NOT NULL, \n\tstate_version BIGINT NOT NULL, \n\tcontent_hash VARCHAR(64) NOT NULL, \n\tprojection_format VARCHAR(64) NOT NULL, \n\tkind TEXT NOT NULL, \n\ttext TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\ttag_keys TEXT NOT NULL, \n\tembedding BLOB, \n\tprofile_fingerprint VARCHAR(64), \n\tembedding_input_hash VARCHAR(64), \n\tPRIMARY KEY (scope_id, artifact_id), \n\tCONSTRAINT ck_pc_atomic_memory_current_revision CHECK (revision > 0), \n\tCONSTRAINT ck_pc_atomic_memory_current_state_version CHECK (state_version >= 0), \n\tCONSTRAINT ck_pc_atomic_memory_current_embedding_metadata CHECK ((embedding IS NULL AND profile_fingerprint IS NULL AND embedding_input_hash IS NULL) OR (embedding IS NOT NULL AND profile_fingerprint IS NOT NULL AND embedding_input_hash IS NOT NULL))\n)", + "CREATE VIRTUAL TABLE pc_atomic_memory_current_fts USING fts5(scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED, state_version UNINDEXED, content_hash UNINDEXED, projection_format UNINDEXED, kind UNINDEXED, text UNINDEXED, searchable_text, tag_keys UNINDEXED, identity_token, tokenize='unicode61')", + "CREATE TABLE pc_connector_checkpoints (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_id VARCHAR(256) NOT NULL, \n\tconnector_name VARCHAR(128) NOT NULL, \n\tconnector_version VARCHAR(128) NOT NULL, \n\tcheckpoint BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, binding_id)\n)", + "CREATE TABLE pc_decision_observations (\n\tscope_id VARCHAR(256) NOT NULL, \n\tobservation_id VARCHAR(128) NOT NULL, \n\toperation_id VARCHAR(256) NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tconsumer VARCHAR(128) NOT NULL, \n\tpolicy_id VARCHAR(256) NOT NULL, \n\tpolicy_version VARCHAR(64) NOT NULL, \n\tmode VARCHAR(16) NOT NULL, \n\tprivacy_boundary VARCHAR(32) NOT NULL, \n\tcoverage VARCHAR(16) NOT NULL, \n\tverdict VARCHAR(16) NOT NULL, \n\tassessment_source VARCHAR(32) NOT NULL, \n\tfinal_action VARCHAR(128) NOT NULL, \n\tused_fallback BOOLEAN NOT NULL, \n\tfallback_reason VARCHAR(128), \n\tprovider_id VARCHAR(128), \n\tbackend_model_id VARCHAR(256), \n\tmodel_policy_id VARCHAR(256), \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT, \n\toutput_tokens BIGINT, \n\tlatency_ms FLOAT, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, observation_id), \n\tCONSTRAINT ck_pc_decision_observations_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_decision_observations_input_tokens_nonnegative CHECK (input_tokens IS NULL OR input_tokens >= 0), \n\tCONSTRAINT ck_pc_decision_observations_output_tokens_nonnegative CHECK (output_tokens IS NULL OR output_tokens >= 0), \n\tCONSTRAINT ck_pc_decision_observations_latency_nonnegative CHECK (latency_ms IS NULL OR latency_ms >= 0)\n)", + "CREATE TABLE pc_dream_runs (\n\tscope_id VARCHAR(256) NOT NULL, \n\trun_id VARCHAR(64) NOT NULL, \n\tprincipal_key VARCHAR(64) NOT NULL, \n\tidempotency_key VARCHAR(128) NOT NULL, \n\trequest_digest VARCHAR(71) NOT NULL, \n\toperation VARCHAR(32) NOT NULL, \n\tstatus VARCHAR(16) NOT NULL, \n\taccepted_at BIGINT NOT NULL, \n\tgeneration INTEGER NOT NULL, \n\trequest_generation BIGINT NOT NULL, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, run_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_dream_idempotency UNIQUE (scope_id, principal_key, idempotency_key), \n\tCONSTRAINT ck_pc_dream_generation CHECK (generation >= 0)\n)", + "CREATE TABLE pc_external_skill_registrations (\n\tscope_id VARCHAR(256) NOT NULL, \n\texternal_skill_id VARCHAR(128) NOT NULL, \n\tprovider VARCHAR(128) NOT NULL, \n\tagent_kind VARCHAR(128) NOT NULL, \n\thost_id VARCHAR(128) NOT NULL, \n\tinstallation_scope VARCHAR(16) NOT NULL, \n\tlocator VARCHAR(2000) NOT NULL, \n\tlocator_hash VARCHAR(64) NOT NULL, \n\tfingerprint VARCHAR(64) NOT NULL, \n\tname VARCHAR(128) NOT NULL, \n\tdescription VARCHAR(2000) NOT NULL, \n\tPRIMARY KEY (scope_id, external_skill_id), \n\tCONSTRAINT uq_pc_external_skill_registrations_binding UNIQUE (scope_id, provider, host_id, installation_scope, locator_hash), \n\tCONSTRAINT ck_pc_external_skill_registrations_scope CHECK (installation_scope IN ('user', 'project', 'plugin'))\n)", + "CREATE TABLE pc_memory_entry_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tmemory_artifact_id VARCHAR(128) NOT NULL, \n\thead_revision INTEGER NOT NULL, \n\tentry_id VARCHAR(128) NOT NULL, \n\tentry_version_id VARCHAR(128) NOT NULL, \n\tentry_content_hash VARCHAR(64) NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tPRIMARY KEY (scope_id, memory_artifact_id, entry_id), \n\tFOREIGN KEY(scope_id, memory_artifact_id, entry_id, entry_version_id) REFERENCES pc_memory_entry_versions (scope_id, memory_artifact_id, entry_id, entry_version_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_memory_entry_heads_revision_positive CHECK (head_revision > 0)\n)", + "CREATE TABLE pc_memory_entry_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tmemory_artifact_id VARCHAR(128) NOT NULL, \n\tentry_id VARCHAR(128) NOT NULL, \n\tentry_version_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tprevious_version_id VARCHAR(128), \n\tkind VARCHAR(128) NOT NULL, \n\ttext TEXT NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\tentry_content_hash VARCHAR(64) NOT NULL, \n\tcreated_in_revision INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, memory_artifact_id, entry_version_id), \n\tCONSTRAINT uq_pc_memory_entry_versions_logical_version UNIQUE (scope_id, memory_artifact_id, entry_id, version), \n\tCONSTRAINT uq_pc_memory_entry_versions_identity UNIQUE (scope_id, memory_artifact_id, entry_id, entry_version_id), \n\tCONSTRAINT ck_pc_memory_entry_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_memory_entry_versions_revision_positive CHECK (created_in_revision > 0)\n)", + "CREATE TABLE pc_memory_source_windows (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\twindow_limit BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tCONSTRAINT ck_pc_memory_source_window_positive CHECK (source_through > 0 AND window_limit > 0)\n)", + "CREATE TABLE pc_model_usage_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpurpose VARCHAR(64) NOT NULL, \n\toperation VARCHAR(16) NOT NULL, \n\trequests BIGINT NOT NULL, \n\tinput_tokens BIGINT NOT NULL, \n\toutput_tokens BIGINT NOT NULL, \n\tinput_complete BOOLEAN NOT NULL, \n\toutput_complete BOOLEAN NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, purpose, operation), \n\tCONSTRAINT ck_pc_model_usage_daily_operation CHECK (operation IN ('generation', 'embedding')), \n\tCONSTRAINT ck_pc_model_usage_daily_requests_nonnegative CHECK (requests >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_input_nonnegative CHECK (input_tokens >= 0), \n\tCONSTRAINT ck_pc_model_usage_daily_output_nonnegative CHECK (output_tokens >= 0)\n)", + "CREATE TABLE pc_profile_policies (\n\tscope_id VARCHAR(256) NOT NULL, \n\tgeneration_enabled BOOLEAN NOT NULL, \n\tactivation_mode VARCHAR(32) NOT NULL, \n\tpending_candidate_id VARCHAR(128), \n\tversion BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(scope_id, pending_candidate_id) REFERENCES pc_artifact_candidate_heads (scope_id, candidate_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_profile_policies_activation_mode CHECK (activation_mode IN ('automatic', 'review_required')), \n\tCONSTRAINT ck_pc_profile_policies_version_positive CHECK (version > 0)\n)", + "CREATE TABLE pc_recall_effort_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\tpolicy_id VARCHAR(128) NOT NULL, \n\tassessment VARCHAR(64) NOT NULL, \n\tpreparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_preparations_nonnegative CHECK (preparations >= 0), \n\trounds BIGINT NOT NULL CONSTRAINT ck_recall_effort_rounds_nonnegative CHECK (rounds >= 0), \n\texpanded_preparations BIGINT NOT NULL CONSTRAINT ck_recall_effort_expanded_preparations_nonnegative CHECK (expanded_preparations >= 0), \n\tadmission_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_admission_expansions_nonnegative CHECK (admission_expansions >= 0), \n\tpolicy_floor_expansions BIGINT NOT NULL CONSTRAINT ck_recall_effort_policy_floor_expansions_nonnegative CHECK (policy_floor_expansions >= 0), \n\tcandidate_round_samples BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidate_round_samples_nonnegative CHECK (candidate_round_samples >= 0), \n\tcandidates_assessed BIGINT NOT NULL CONSTRAINT ck_recall_effort_candidates_assessed_nonnegative CHECK (candidates_assessed >= 0), \n\tfinal_candidate_pool BIGINT NOT NULL CONSTRAINT ck_recall_effort_final_candidate_pool_nonnegative CHECK (final_candidate_pool >= 0), \n\tadded_embeddings BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_embeddings_nonnegative CHECK (added_embeddings >= 0), \n\tadded_generation_calls BIGINT NOT NULL CONSTRAINT ck_recall_effort_added_generation_calls_nonnegative CHECK (added_generation_calls >= 0), \n\ttruncated_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_truncated_items_nonnegative CHECK (truncated_items >= 0), \n\tdropped_items BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_items_nonnegative CHECK (dropped_items >= 0), \n\tdropped_below_min_bytes BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_below_min_bytes_nonnegative CHECK (dropped_below_min_bytes >= 0), \n\tdropped_no_fitting_truncation BIGINT NOT NULL CONSTRAINT ck_recall_effort_dropped_no_fitting_truncation_nonnegative CHECK (dropped_no_fitting_truncation >= 0), \n\tPRIMARY KEY (scope_id, usage_date, policy_id, assessment), \n\tCONSTRAINT ck_pc_recall_effort_daily_drop_total CHECK (dropped_items = dropped_below_min_bytes + dropped_no_fitting_truncation)\n)", + "CREATE TABLE pc_recall_token_daily (\n\tscope_id VARCHAR(256) NOT NULL, \n\tusage_date DATE NOT NULL, \n\testimator_id VARCHAR(128) NOT NULL, \n\testimator_version VARCHAR(64) NOT NULL, \n\tpreparations BIGINT NOT NULL, \n\tready_preparations BIGINT NOT NULL, \n\tcomparable_preparations BIGINT NOT NULL, \n\tbaseline_tokens BIGINT NOT NULL, \n\trecalled_tokens BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, usage_date, estimator_id, estimator_version), \n\tCONSTRAINT ck_pc_recall_token_daily_preparations_nonnegative CHECK (preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_nonnegative CHECK (ready_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_nonnegative CHECK (comparable_preparations >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_comparable_ready CHECK (comparable_preparations <= ready_preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_ready_total CHECK (ready_preparations <= preparations), \n\tCONSTRAINT ck_pc_recall_token_daily_baseline_nonnegative CHECK (baseline_tokens >= 0), \n\tCONSTRAINT ck_pc_recall_token_daily_recalled_nonnegative CHECK (recalled_tokens >= 0)\n)", + "CREATE TABLE pc_receipt_migration_review (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\treason VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, source_id)\n)", + "CREATE TABLE pc_recurrence_match (\n\tscope_id VARCHAR(256) NOT NULL, \n\tmatch_key VARCHAR(71) NOT NULL, \n\ttask_outcome_source_type VARCHAR(128) NOT NULL, \n\ttask_outcome_source_id VARCHAR(256) NOT NULL, \n\ttask_outcome_position BIGINT NOT NULL, \n\tfailure_item_kind VARCHAR(16) NOT NULL, \n\tfailure_item_index INTEGER NOT NULL, \n\tfailure_item_digest VARCHAR(71) NOT NULL, \n\tcandidate_set_mode VARCHAR(32) NOT NULL, \n\tcandidate_set_digest VARCHAR(71) NOT NULL, \n\tresult VARCHAR(16) NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\tsignature_key TEXT, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, match_key), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_recurrence_match_identity UNIQUE (scope_id, task_outcome_source_type, task_outcome_source_id, failure_item_kind, failure_item_index), \n\tCONSTRAINT ck_pc_recurrence_match_result CHECK (result IN ('matched', 'unmatched', 'ambiguous')), \n\tCONSTRAINT ck_pc_recurrence_match_candidate_set_mode CHECK (candidate_set_mode IN ('handoff_citations', 'scope_heads')), \n\tCONSTRAINT ck_pc_recurrence_match_failure_item_kind CHECK (failure_item_kind IN ('observation', 'check')), \n\tCONSTRAINT ck_pc_recurrence_match_position_positive CHECK (task_outcome_position > 0), \n\tCONSTRAINT ck_pc_recurrence_match_item_index_nonnegative CHECK (failure_item_index >= 0)\n)", + "CREATE TABLE pc_recurrence_observation (\n\tscope_id VARCHAR(256) NOT NULL, \n\tobservation_id VARCHAR(71) NOT NULL, \n\tselection_key VARCHAR(71) NOT NULL, \n\tverdict_key VARCHAR(71) NOT NULL, \n\tevent VARCHAR(16) NOT NULL, \n\tmatch_basis VARCHAR(8) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsignature_key TEXT NOT NULL, \n\tsignature_key_hash BLOB NOT NULL, \n\ttask_outcome_source_type VARCHAR(128) NOT NULL, \n\ttask_outcome_source_id VARCHAR(256) NOT NULL, \n\ttask_outcome_position BIGINT NOT NULL, \n\tpayload BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, observation_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_recurrence_observation_selection UNIQUE (scope_id, selection_key), \n\tCONSTRAINT uq_pc_recurrence_observation_verdict UNIQUE (scope_id, verdict_key), \n\tCONSTRAINT ck_pc_recurrence_observation_event CHECK (event IN ('selected', 'recurred', 'avoided')), \n\tCONSTRAINT ck_pc_recurrence_observation_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_recurrence_observation_position_positive CHECK (task_outcome_position > 0)\n)", + "CREATE TABLE pc_scope_bindings (\n\tintegration VARCHAR(128) NOT NULL, \n\tkind VARCHAR(64) NOT NULL, \n\texternal_id VARCHAR(256) NOT NULL, \n\texternal_id_search TEXT NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (integration, kind, external_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)", + "CREATE TABLE pc_scope_context_references (\n\tscope_id VARCHAR(256) NOT NULL, \n\treferenced_scope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (scope_id, referenced_scope_id), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tFOREIGN KEY(referenced_scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_scope_context_references_not_self CHECK (scope_id <> referenced_scope_id)\n)", + "CREATE TABLE pc_scope_creation_requests (\n\tidempotency_key VARCHAR(256) NOT NULL, \n\trequest_digest VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (idempotency_key), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)", + "CREATE TABLE pc_scope_external_references (\n\tscope_id VARCHAR(256) NOT NULL, \n\tordinal INTEGER NOT NULL, \n\tkind VARCHAR(128) NOT NULL, \n\tvalue VARCHAR(2000) NOT NULL, \n\tvalue_search TEXT NOT NULL, \n\tvalue_digest VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, ordinal), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE CASCADE, \n\tCONSTRAINT uq_pc_scope_external_references_value UNIQUE (scope_id, kind, value_digest), \n\tCONSTRAINT ck_pc_scope_external_references_ordinal_nonnegative CHECK (ordinal >= 0)\n)", + "CREATE TABLE pc_scope_settings (\n\tname VARCHAR(64) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\tPRIMARY KEY (name), \n\tFOREIGN KEY(scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT\n)", + "CREATE TABLE pc_scopes (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttitle VARCHAR(256) NOT NULL, \n\tsummary VARCHAR(2000) NOT NULL, \n\tscope_id_search TEXT NOT NULL, \n\ttitle_search TEXT NOT NULL, \n\tsummary_search TEXT NOT NULL, \n\tparent_scope_id VARCHAR(256), \n\tversion INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tFOREIGN KEY(parent_scope_id) REFERENCES pc_scopes (scope_id) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_scopes_version_positive CHECK (version > 0)\n)", + "CREATE TABLE pc_server_identity (\n\tsingleton_key INTEGER NOT NULL, \n\tserver_id VARCHAR(36) NOT NULL, \n\tPRIMARY KEY (singleton_key), \n\tCONSTRAINT ck_pc_server_identity_singleton CHECK (singleton_key = 1), \n\tUNIQUE (server_id)\n)", + "CREATE TABLE pc_skill_packages (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttree_digest VARCHAR(64) NOT NULL, \n\tarchive_digest VARCHAR(64) NOT NULL, \n\tarchive_bytes BLOB NOT NULL, \n\tmanifest BLOB NOT NULL, \n\tfile_count INTEGER NOT NULL, \n\tuncompressed_size BIGINT NOT NULL, \n\tarchive_size BIGINT NOT NULL, \n\tcreated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, tree_digest), \n\tCONSTRAINT ck_pc_skill_packages_file_count CHECK (file_count > 0 AND file_count <= 256), \n\tCONSTRAINT ck_pc_skill_packages_uncompressed_size CHECK (uncompressed_size > 0 AND uncompressed_size <= 4194304), \n\tCONSTRAINT ck_pc_skill_packages_archive_size CHECK (archive_size > 0 AND archive_size <= 5242880)\n)", + "CREATE TABLE pc_skill_publications (\n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_id VARCHAR(64) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\tdesired_state VARCHAR(16) DEFAULT 'published' NOT NULL, \n\tdesired_revision INTEGER NOT NULL, \n\tdesired_tree_digest VARCHAR(64) NOT NULL, \n\tobserved_revision INTEGER, \n\tobserved_tree_digest VARCHAR(64), \n\tobserved_generation BIGINT, \n\tdestination TEXT, \n\tstate VARCHAR(32) NOT NULL, \n\tselected_runtime_variant VARCHAR(128), \n\tenvironment_fingerprint VARCHAR(64), \n\tlast_error_code VARCHAR(128), \n\tobserved_at DATETIME, \n\tgeneration BIGINT NOT NULL, \n\tupdated_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, target_id, artifact_id), \n\tCONSTRAINT ck_pc_skill_publications_desired_revision_positive CHECK (desired_revision > 0), \n\tCONSTRAINT ck_pc_skill_publications_observed_revision_positive CHECK (observed_revision IS NULL OR observed_revision > 0), \n\tCONSTRAINT ck_pc_skill_publications_desired_state CHECK (desired_state IN ('published', 'unpublished')), \n\tCONSTRAINT ck_pc_skill_publications_state CHECK (state IN ('unpublished', 'pending', 'current', 'update_available', 'delivery_failed', 'conflict', 'drifted', 'incompatible')), \n\tCONSTRAINT ck_pc_skill_publications_observed_generation_nonnegative CHECK (observed_generation IS NULL OR observed_generation >= 0), \n\tCONSTRAINT ck_pc_skill_publications_generation_nonnegative CHECK (generation >= 0)\n)", + "CREATE TABLE pc_source_cursors (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tcursor BLOB NOT NULL, \n\tgeneration BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, binding_name), \n\tCONSTRAINT ck_pc_source_cursors_generation_nonnegative CHECK (generation >= 0)\n)", + "CREATE TABLE pc_source_definition_manifests (\n\tdefinition_name VARCHAR(128) NOT NULL, \n\tdefinition_version VARCHAR(128) NOT NULL, \n\tfingerprint VARCHAR(71) NOT NULL, \n\tmanifest BLOB NOT NULL, \n\tPRIMARY KEY (definition_name, definition_version), \n\tCONSTRAINT uq_pc_source_definition_manifest_fingerprint UNIQUE (definition_name, fingerprint)\n)", + "CREATE TABLE pc_source_journal_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tposition BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id), \n\tCONSTRAINT ck_pc_source_journal_heads_position_nonnegative CHECK (position >= 0)\n)", + "CREATE TABLE pc_sources (\n\tscope_id VARCHAR(256) NOT NULL, \n\tsource_type VARCHAR(128) NOT NULL, \n\tsource_id VARCHAR(256) NOT NULL, \n\tpayload BLOB NOT NULL, \n\tjournal_position BIGINT NOT NULL, \n\tPRIMARY KEY (scope_id, source_type, source_id), \n\tCONSTRAINT uq_pc_sources_scope_journal_position UNIQUE (scope_id, journal_position)\n)", + "CREATE TABLE pc_topic_memory_active_chunks (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tchunk_ordinal INTEGER NOT NULL, \n\tstart_offset INTEGER NOT NULL, \n\tend_offset INTEGER NOT NULL, \n\tchunk_text TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tpolicy_version VARCHAR(32) NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id, chunk_ordinal), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_topic_memory_revision_publications (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_ordinal_nonnegative CHECK (chunk_ordinal >= 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_start_nonnegative CHECK (start_offset >= 0), \n\tCONSTRAINT ck_pc_topic_memory_active_chunks_offsets CHECK (end_offset > start_offset)\n)", + "CREATE TABLE pc_topic_memory_active_topics (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\ttitle TEXT NOT NULL, \n\tsummary TEXT NOT NULL, \n\tsearchable_text TEXT NOT NULL, \n\tsource_count INTEGER NOT NULL, \n\tPRIMARY KEY (scope_id, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_topic_memory_revision_publications (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_active_topics_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_active_topics_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_topic_memory_active_topics_sources_nonnegative CHECK (source_count >= 0)\n)", + "CREATE VIRTUAL TABLE pc_topic_memory_chunk_fts USING fts5(\n scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED,\n chunk_ordinal UNINDEXED, start_offset UNINDEXED,\n title UNINDEXED, summary UNINDEXED, chunk_text UNINDEXED,\n searchable_text, tokenize='unicode61'\n)", + "CREATE TABLE pc_topic_memory_fts_index (\n\tsingleton INTEGER NOT NULL, \n\tschema_version INTEGER NOT NULL, \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_topic_memory_fts_index_singleton CHECK (singleton = 1)\n)", + "CREATE TABLE pc_topic_memory_processing_targets (\n\tbinding_name VARCHAR(128) NOT NULL, \n\tscope_id VARCHAR(256) NOT NULL, \n\ttarget_request_generation BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tcaptured_flush_generation BIGINT NOT NULL, \n\tobserved_dirty_generation BIGINT NOT NULL, \n\tPRIMARY KEY (binding_name, scope_id), \n\tCONSTRAINT ck_pc_topic_processing_target CHECK (target_request_generation > 0 AND source_through >= 0 AND captured_flush_generation >= 0 AND observed_dirty_generation >= 0)\n)", + "CREATE TABLE pc_topic_memory_retrieval_shape (\n\tsingleton INTEGER NOT NULL, \n\tshape VARCHAR(16) NOT NULL, \n\tprofile_fingerprint VARCHAR(64), \n\tPRIMARY KEY (singleton), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_singleton CHECK (singleton = 1), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_value CHECK (shape IN ('fts', 'hybrid')), \n\tCONSTRAINT ck_pc_topic_memory_retrieval_shape_profile CHECK ((shape = 'fts' AND profile_fingerprint IS NULL) OR (shape = 'hybrid' AND profile_fingerprint IS NOT NULL))\n)", + "CREATE TABLE pc_topic_memory_revision_publications (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tpublished_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_topic_memory_publications_family CHECK (family = 'topic-memory'), \n\tCONSTRAINT ck_pc_topic_memory_publications_revision_positive CHECK (revision > 0)\n)", + "CREATE VIRTUAL TABLE pc_topic_memory_topic_fts USING fts5(\n scope_id UNINDEXED, artifact_id UNINDEXED, revision UNINDEXED,\n title UNINDEXED, summary UNINDEXED, searchable_text, tokenize='unicode61'\n)", + "CREATE TABLE pc_topic_memory_work_budgets (\n\tscope_id VARCHAR(256) NOT NULL, \n\tbinding_name VARCHAR(128) NOT NULL, \n\tsource_after BIGINT NOT NULL, \n\tsource_through BIGINT NOT NULL, \n\tattempt_id VARCHAR(36) NOT NULL, \n\tattempts INTEGER NOT NULL, \n\trequests BIGINT NOT NULL, \n\ttokens BIGINT NOT NULL, \n\tfailure_code VARCHAR(64) NOT NULL, \n\tPRIMARY KEY (scope_id, binding_name, source_after), \n\tCONSTRAINT ck_pc_topic_budget_window CHECK (source_after >= 0 AND source_through > source_after), \n\tCONSTRAINT ck_pc_topic_budget_usage CHECK (attempts > 0 AND requests >= 0 AND tokens >= 0)\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_decision_observations_operation ON pc_decision_observations (scope_id, operation_id, created_at)", + "CREATE INDEX ix_pc_decision_observations_scope_created ON pc_decision_observations (scope_id, created_at, observation_id)", + "CREATE INDEX ix_pc_dream_dispatch ON pc_dream_runs (scope_id, operation, status, request_generation)", + "CREATE INDEX ix_pc_dream_list ON pc_dream_runs (scope_id, accepted_at, run_id)", + "CREATE INDEX ix_pc_processing_intent_binding_sequence ON pc_artifact_processing_intents (binding_name, pending_sequence)", + "CREATE INDEX ix_pc_recurrence_observation_key ON pc_recurrence_observation (scope_id, signature_key_hash)", + "CREATE INDEX ix_pc_recurrence_observation_outcome ON pc_recurrence_observation (scope_id, task_outcome_source_type, task_outcome_source_id, task_outcome_position)", + "CREATE INDEX ix_pc_recurrence_observation_revision ON pc_recurrence_observation (scope_id, family, artifact_id, revision)", + "CREATE TRIGGER pc_atomic_memory_fts_delete AFTER DELETE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_fts WHERE identity_token MATCH ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); END", + "CREATE TRIGGER pc_atomic_memory_fts_insert AFTER INSERT ON pc_atomic_memory_current BEGIN INSERT INTO pc_atomic_memory_current_fts (scope_id, artifact_id, revision, state_version, content_hash, projection_format, kind, text, searchable_text, tag_keys, identity_token) VALUES (new.scope_id, new.artifact_id, new.revision, new.state_version, new.content_hash, new.projection_format, new.kind, new.text, new.searchable_text, new.tag_keys, 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id))); END", + "CREATE TRIGGER pc_atomic_memory_fts_update AFTER UPDATE ON pc_atomic_memory_current BEGIN DELETE FROM pc_atomic_memory_current_fts WHERE identity_token MATCH ('i' || lower(hex(old.scope_id)) || 'x' || lower(hex(old.artifact_id))); INSERT INTO pc_atomic_memory_current_fts (scope_id, artifact_id, revision, state_version, content_hash, projection_format, kind, text, searchable_text, tag_keys, identity_token) VALUES (new.scope_id, new.artifact_id, new.revision, new.state_version, new.content_hash, new.projection_format, new.kind, new.text, new.searchable_text, new.tag_keys, 'i' || lower(hex(new.scope_id)) || 'x' || lower(hex(new.artifact_id))); END" + ], + "sqlite_expand": [ + "ALTER TABLE pc_scopes ADD COLUMN atomic_memory_directory_generation BIGINT NOT NULL DEFAULT 0 CONSTRAINT ck_pc_scopes_atomic_directory_generation CHECK (atomic_memory_directory_generation >= 0)", + "ALTER TABLE pc_artifact_heads ADD COLUMN content_kind VARCHAR(128) COLLATE BINARY" + ], + "content_schema": { + "$defs": { + "AtomicMemoryCreation": { + "additionalProperties": false, + "description": "Select merge inputs from the exact lineage of revision one.", + "properties": { + "type": { + "const": "merge", + "default": "merge", + "title": "Type", + "type": "string" + }, + "input_artifact_ids": { + "items": { + "type": "string" + }, + "minItems": 2, + "title": "Input Artifact Ids", + "type": "array" + } + }, + "required": [ + "input_artifact_ids" + ], + "title": "AtomicMemoryCreation", + "type": "object" + } + }, + "additionalProperties": false, + "description": "One durable fact, preference or other atomic memory.", + "properties": { + "schema": { + "const": "powercontext.atomic-memory.v1", + "default": "powercontext.atomic-memory.v1", + "title": "Schema", + "type": "string" + }, + "kind": { + "maxLength": 128, + "minLength": 1, + "title": "Kind", + "type": "string" + }, + "text": { + "minLength": 1, + "title": "Text", + "type": "string" + }, + "creation": { + "anyOf": [ + { + "$ref": "#/$defs/AtomicMemoryCreation" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "kind", + "text" + ], + "title": "AtomicMemoryContent", + "type": "object" + } +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/tasks/atomic_directory_v1.py b/src/powercontext/builtin/persistence/migrations/resources/tasks/atomic_directory_v1.py new file mode 100644 index 0000000000..f8d9134e2b --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/tasks/atomic_directory_v1.py @@ -0,0 +1,158 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Immutable current-head directory task; the shared executor owns commits and recovery. + +This resource imports no current Repository, Runtime or domain decoder. Its v1 +content schema and supplementary validation travel with the reviewed bundle. +""" + +import json +import unicodedata +from pathlib import Path + +from jsonschema import Draft202012Validator, ValidationError + +_MAX_GENERATION = (1 << 63) - 1 +_BATCH_SIZE = 200 +_RESOURCE = json.loads((Path(__file__).parents[1] / "schemas/atomic_directory_sqlite_v1.json").read_text()) +_CONTENT = Draft202012Validator(_RESOURCE["content_schema"]) + + +def _valid_identity(value, maximum): + return isinstance(value, str) and 0 < len(value) <= maximum and bool(value.strip()) + + +def _check_authority(connection): + orphan = connection.exec_driver_sql( + "SELECT 1 FROM pc_artifact_heads h LEFT JOIN pc_scopes s ON s.scope_id=h.scope_id " + "WHERE h.family='atomic-memory' AND s.scope_id IS NULL LIMIT 1" + ).first() + legacy = connection.exec_driver_sql("SELECT 1 FROM pc_artifacts WHERE family='memory' LIMIT 1").first() + if orphan is not None or legacy is not None: + raise ValueError("atomic_directory_authority_invalid") + after = None + while True: + statement = ( + "SELECT scope_id, atomic_memory_directory_generation FROM pc_scopes ORDER BY scope_id LIMIT 200" + if after is None + else "SELECT scope_id, atomic_memory_directory_generation FROM pc_scopes " + "WHERE scope_id>? ORDER BY scope_id LIMIT 200" + ) + rows = connection.exec_driver_sql( + statement, + () if after is None else (after,), + ).all() + if not rows: + return + for scope_id, generation in rows: + if ( + not _valid_identity(scope_id, 256) + or type(generation) is not int + or not 0 <= generation <= _MAX_GENERATION + ): + raise ValueError("atomic_directory_generation_invalid") + after = rows[-1][0] + + +def _batches(connection): + after = None + while True: + clause = "" if after is None else " AND (h.scope_id,h.artifact_id)>(?,?)" + rows = connection.exec_driver_sql( + "SELECT h.scope_id,h.artifact_id,h.revision,h.content_kind,h.lifecycle_state," + "h.governance_generation,h.merged_into_id,h.replacement_artifact_id,a.content " + "FROM pc_artifact_heads h LEFT JOIN pc_artifacts a " + "ON a.scope_id=h.scope_id AND a.family=h.family AND a.artifact_id=h.artifact_id AND a.revision=h.revision " + "WHERE h.family='atomic-memory'" + clause + " ORDER BY h.scope_id,h.artifact_id LIMIT 201", + () if after is None else after, + ).all() + if not rows: + return + yield rows[:_BATCH_SIZE] + if len(rows) <= _BATCH_SIZE: + return + after = (rows[_BATCH_SIZE - 1][0], rows[_BATCH_SIZE - 1][1]) + + +def _kind(row): + scope_id, identity, revision, _kind_value, lifecycle, generation, merged, replacement, body = row + try: + content = json.loads(body) + _CONTENT.validate(content) + kind = content["kind"] + text = content["text"] + if ( + not kind.strip() + or not text.strip() + or min(len(text.encode("utf-8")), len(unicodedata.normalize("NFC", text).encode("utf-8"))) > 8192 + or not _valid_identity(scope_id, 256) + or not _valid_identity(identity, 128) + or type(revision) is not int + or revision < 1 + or type(generation) is not int + or generation < 0 + or lifecycle not in {"active", "deprecated", "retired"} + or replacement is not None + or ( + merged is not None + and (not _valid_identity(merged, 128) or lifecycle != "deprecated" or merged == identity) + ) + ): + raise ValueError("invalid current Atomic metadata") # noqa: TRY003, TRY301 + creation = content.get("creation") + if creation is not None: + inputs = creation["input_artifact_ids"] + if len(set(inputs)) != len(inputs) or any(not _valid_identity(item, 128) for item in inputs): + raise ValueError("invalid Atomic creation inputs") # noqa: TRY003, TRY301 + except (TypeError, ValueError, ValidationError) as error: + # Expose only a stable classification, never private content or identities. + raise ValueError("atomic_directory_data_invalid") from error + return kind + + +def verify(connection): + _check_authority(connection) + for rows in _batches(connection): + for row in rows: + if row[3] != _kind(row): + return False + return True + + +def apply_batches(connection): + _check_authority(connection) + for rows in _batches(connection): + # Validate this entire batch before issuing any updates. + changes = [(row, _kind(row)) for row in rows] + changes = [(row, kind) for row, kind in changes if row[3] != kind] + scopes = sorted({row[0] for row, _ in changes}) + for scope_id in scopes: + result = connection.exec_driver_sql( + "UPDATE pc_scopes SET atomic_memory_directory_generation=atomic_memory_directory_generation+1 " + "WHERE scope_id=? AND atomic_memory_directory_generation>=0 AND atomic_memory_directory_generation None: + connection = op.get_bind() + if connection.dialect.name != "sqlite": + raise ValueError("This registered target supports only SQLite") # noqa: TRY003 + # The earlier four-table acceptance history is usable for empty initialization + # only. Existing real business layouts are explicitly adopted at p0004. + tables = ("pc_artifact_tags", "pc_artifact_candidate_versions", "pc_artifact_heads", "pc_artifacts") + for table in tables: + if connection.exec_driver_sql(f"SELECT 1 FROM {table} LIMIT 1").first() is not None: # noqa: S608 + raise ValueError("Nonempty acceptance layouts require a registered legacy adapter") # noqa: TRY003 + for table in tables: + connection.exec_driver_sql(f"DROP TABLE {table}") + resource = json.loads((Path(__file__).parents[1] / "schemas/atomic_directory_sqlite_v1.json").read_text()) + for statement in resource["sqlite_init"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_directory.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_directory.py new file mode 100644 index 0000000000..522fd2f3db --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0005_atomic_directory.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Expand directory metadata; explicit data tasks run under the same maintenance lock.""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0005" +down_revision = "p0004" + + +def upgrade() -> None: + connection = op.get_bind() + if connection.dialect.name != "sqlite": + raise ValueError("This registered target supports only SQLite") # noqa: TRY003 + resource = json.loads((Path(__file__).parents[1] / "schemas/atomic_directory_sqlite_v1.json").read_text()) + for statement in resource["sqlite_expand"]: + connection.exec_driver_sql(statement) diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index 2664f2f440..ddee2f531e 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -146,6 +146,13 @@ def verify(self) -> MigrationResult: raise MigrationError(plan.state, "Inspect and explicitly apply the registered bundle before verification.") return MigrationResult(revision=self.bundle.head, changed=False) + def assert_business_startup_allowed(self) -> None: + """Reject an unfinished recorded maintenance window before any initializer writes.""" + + evidence = self._evidence() + if evidence is not None and evidence["state"] != "complete": + raise MigrationError("recovery_required", "Complete explicit maintenance before business startup.") + def apply( self, *, @@ -191,7 +198,21 @@ def apply( connection.commit() self._upgrade(connection) self._begin(connection) + for task in self.bundle.data_tasks: + functions = self.bundle.task_functions(task) + try: + for _ in functions["apply_batches"](connection): + connection.commit() + self._begin(connection) + except ValueError as error: + raise MigrationError( + "verification_failed", "A frozen data task rejected the current data." + ) from error self._verify_schema(connection, self.bundle.head) + if self.bundle.pending_data_tasks(connection, self.bundle.head): + raise MigrationError( + "verification_failed", "Required data tasks did not satisfy their postconditions." + ) connection.commit() evidence["state"] = "complete" evidence["release_checksums"] = self.bundle.checksums @@ -262,7 +283,7 @@ def _connect(self, *, read_only: bool) -> Iterator[Connection]: finally: engine.dispose() - def _inspect(self, connection: Connection, **options: Any) -> MigrationPlan: + def _inspect(self, connection: Connection, **options: Any) -> MigrationPlan: # noqa: C901 inventory = sqlite_inventory(connection) controls = [row for row in inventory if row[1] in _CONTROL_NAMES or row[2] in _CONTROL_NAMES] initialized = bool(controls) @@ -277,6 +298,17 @@ def _inspect(self, connection: Connection, **options: Any) -> MigrationPlan: raise MigrationError("incompatible_schema", "Multiple revision heads are unsupported.") revision = revisions[0] if revisions else None self._verify_schema(connection, revision) + minimum = self.bundle.minimum_source_revision + if ( + minimum is not None + and revision is not None + and self.bundle.revisions.index(revision) < self.bundle.revisions.index(minimum) + ): + evidence = self._evidence() + if evidence is None or evidence["state"] == "complete" or evidence["source_revision"] is not None: + raise MigrationError( + "unsupported_target", "This target does not adopt a historical acceptance database." + ) if self.bundle.pending(revision) and self._evidence() is None: raise MigrationError( "recovery_required", @@ -290,9 +322,28 @@ def _inspect(self, connection: Connection, **options: Any) -> MigrationPlan: self._verify_integrity(connection) else: revision = None - return self._plan(revision, fingerprint, initialized=initialized, **options) + try: + data_tasks = self.bundle.pending_data_tasks(connection, revision) + except MigrationError as error: + evidence = self._evidence() + if error.code == "verification_failed" and evidence and evidence["state"] != "complete": + raise MigrationError( + "recovery_required", "Repair rejected data within the original maintenance window." + ) from error + raise + if initialized and data_tasks and self._evidence() is None: + raise MigrationError( + "recovery_required", "Original evidence is required before resuming pending data tasks." + ) + return self._plan( + revision, + fingerprint, + initialized=initialized, + data_tasks=data_tasks, + **options, + ) - def _plan( + def _plan( # noqa: C901 - one authority for schema/data/evidence plan classification self, revision: str | None, fingerprint: str, @@ -301,10 +352,13 @@ def _plan( backup_policy: BackupPolicy, service: dict[str, Any] | None, shared_database: bool, + data_tasks: tuple[str, ...] | None = None, ) -> MigrationPlan: pending = self.bundle.pending(revision) + if data_tasks is None: + data_tasks = tuple(task["name"] for task in self.bundle.data_tasks if task["revision"] in pending) adopt = not initialized and revision is not None - state = "migration_required" if pending or adopt else "ready" + state = "migration_required" if pending or adopt or data_tasks else "ready" if not initialized and revision is None: state = "uninitialized" evidence = self._evidence() @@ -340,6 +394,7 @@ def _plan( "configuration_digest": self.configuration_digest, "state": state, "revisions": pending, + "data_tasks": data_tasks, "adopt_baseline": adopt, "backup_policy": backup_policy, "backup_available": available, @@ -508,7 +563,12 @@ def _upgrade(self, connection: Connection) -> None: self._begin(connection) self._verify_schema(connection, current) with ALEMBIC_CONTEXT_LOCK: - command.upgrade(self.bundle.config(connection), revision) + try: + command.upgrade(self.bundle.config(connection), revision) + except ValueError as error: + raise MigrationError( + "verification_failed", "A frozen revision rejected its source layout." + ) from error self._verify_schema(connection, revision) connection.commit() current = revision diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 6415d3e35c..9f80414aa6 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -97,6 +97,10 @@ async def open( ) -> AsyncIterator[SQLiteProfile]: """Create, initialize and exclusively own one SQLite engine.""" + if config.is_persistent: + from powercontext.builtin.persistence.migrations.deployment import assert_directory_maintenance_complete + + await asyncio.to_thread(assert_directory_maintenance_complete, config) _create_database_directory(config.url) engine_options: dict[str, object] = {"echo": config.echo, "hide_parameters": True} if not config.is_persistent: diff --git a/src/powercontext/builtin/persistence/tags.py b/src/powercontext/builtin/persistence/tags.py index ee44b55be0..0e9e460498 100644 --- a/src/powercontext/builtin/persistence/tags.py +++ b/src/powercontext/builtin/persistence/tags.py @@ -136,11 +136,14 @@ async def get(self, scope_id: str, target: TagTarget) -> ArtifactTagSet: current = await self._read(connection, scope_id, target) return tag_set(scope_id, target, current.tags) - async def replace( + async def replace( # noqa: C901 self, scope_id: str, target: TagTarget, tags: tuple[str, ...], *, expected_etag: str, execution_context=None ) -> ArtifactTagSet: desired = normalize_tags(tags) async with self._database.transaction() as connection: + directory = self._artifacts.atomic_memory_directory if target.family == "atomic-memory" else None + if directory is not None: + await directory.lock_scope(connection, scope_id) # Acquire the database write lock before any reads. In particular, # SELECT FOR UPDATE alone cannot serialize empty-set writes on SQLite. locked = await connection.execute( @@ -186,6 +189,8 @@ async def replace( .where(_where(identity), ARTIFACT_TAGS_TABLE.c.tag_key == key) .values(tag=label) ) + if directory is not None and previous != desired: + await directory.record_mutation(connection, scope_id) if target.family == "atomic-memory" and self._projection_hook is not None: await self._projection_hook(connection, scope_id, target.artifact_id, tuple(desired)) return tag_set(scope_id, target, tags) diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index 8dbe0cfa0d..58d7a1ff9b 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -2514,6 +2514,10 @@ async def list(self, **kwargs): async with self._runtime._scope_operation(self.scope_id): return await self._scoped.list(**kwargs) + async def query_directory(self, **kwargs): + async with self._runtime._scope_operation(self.scope_id): + return await self._scoped.query_directory(**kwargs) + async def search(self, query: str, *, _trace: bool = True, **kwargs): async with self._runtime._scoped_operation( self.scope_id, diff --git a/src/powercontext/builtin/runtime/atomic_memory.py b/src/powercontext/builtin/runtime/atomic_memory.py index fbe12d3ecc..7d02eeb085 100644 --- a/src/powercontext/builtin/runtime/atomic_memory.py +++ b/src/powercontext/builtin/runtime/atomic_memory.py @@ -24,7 +24,7 @@ from typing import cast from uuid import uuid4 -from sqlalchemy import select +from sqlalchemy import select, true from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.artifacts.search import ArtifactSearchExecutionContext @@ -56,6 +56,12 @@ ) from powercontext.builtin.persistence.artifacts import ArtifactRepository from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository, atomic_memory_state_expression +from powercontext.builtin.persistence.atomic_memory_directory import ( + AtomicMemoryDirectoryAuthorizationUnavailableError, + AtomicMemoryDirectoryPage, + AtomicMemoryDirectoryQuery, + AtomicMemoryDirectoryUnavailableError, +) from powercontext.builtin.persistence.atomic_memory_index import ( AtomicMemoryIndex, AtomicMemoryIndexError, @@ -88,6 +94,32 @@ async def filters( await self.authorize(connection, scope_id, context, "read") return AtomicMemoryIndexFilter(tag_filter=tags) + async def directory_context( + self, connection: AsyncConnection, scope_id: str, context: ArtifactSearchExecutionContext | None + ) -> str: + from powercontext.server.authz import AccessDeniedError, AccessUnavailableError + + # This additive API requires an explicit host-owned context, including + # local calls. Existing full-record operations retain their local policy. + if context is None: + raise AccessDeniedError + if context.access is None: + if not context.trusted_local: + raise AccessDeniedError + return "trusted-local-v1" + try: + return await context.access.require_scope_directory_read( + context.principal, scope_id, connection=connection, context=context.audit + ) + except AccessUnavailableError as error: + if error.code in { + "safe_resource_filtering_unavailable", + "transactional_relationships_unavailable", + "relationship_management_unavailable", + }: + raise AtomicMemoryDirectoryAuthorizationUnavailableError from error + raise + def deferred_decision_audit(context: ArtifactSearchExecutionContext | None): """Flush decision audit only after the business snapshot closes.""" @@ -177,7 +209,7 @@ def __init__( ) self.service = AtomicMemoryService( artifacts=artifacts, - states=AtomicMemoryStateRepository(), + states=AtomicMemoryStateRepository(atomic_memory_directory=artifacts.atomic_memory_directory), security=self.security, projections=self.publisher, merge_tags=self.merge_tags, @@ -196,6 +228,53 @@ def __init__(self, application: AtomicMemoryApplication, scope_id: str) -> None: self.application = application self.scope_id = scope_id + async def query_directory( + self, + *, + states: tuple[str, ...] = ("active",), + kind: str | None = None, + tag_filter: TagFilter | None = None, + limit: int = 50, + cursor: str | None = None, + context: ArtifactSearchExecutionContext | None = None, + connection: AsyncConnection | None = None, + ) -> AtomicMemoryDirectoryPage: + """Read compact current references after qualifying every page's access. + + Directory maintenance must be composed into Artifact writes only after + unified schema/data verification. An uncomposed application fails closed. + A supplied connection retains its caller's consistent transaction; the + caller also defers decision audit until that transaction closes. + """ + + query = AtomicMemoryDirectoryQuery.model_validate({ + "scope_id": self.scope_id, + "states": states, + "kind": kind, + "tag_filter": tag_filter, + "limit": limit, + "cursor": cursor, + }) + async with ( + deferred_decision_audit(context), + self.application.database.transaction(consistent_snapshot=True) + if connection is None + else self.application.database.connection(connection) as connection, + ): + policy_context = await self.application.security.directory_context(connection, self.scope_id, context) + directory = self.application.artifacts.atomic_memory_directory + if directory is None: + raise AtomicMemoryDirectoryUnavailableError + return await directory.query( + connection, + query, + subject=self.application.security.subject(context), + policy_context=policy_context, + # Qualified Scope read and explicit trusted-local both permit + # every row. Per-Artifact-only grants are not a supported mode. + eligibility=true(), + ) + async def get(self, artifact_id: str, *, revision: int | None = None, context=None) -> AtomicMemoryRecord: # Builtin authority shares the business snapshot. Configured external # providers keep their own decision boundary. Audit writes are flushed diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index e251ea2557..ea8b6b07a4 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import json import logging import os @@ -95,12 +96,15 @@ UsageReportingEmbeddingModel, UsageReportingStructuredGenerator, ) +from powercontext.builtin.persistence.atomic_memory_directory import AtomicMemoryDirectoryRepository from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexCapabilities from powercontext.builtin.persistence.atomic_memory_readiness import ( assert_atomic_memory_migration_ready, ensure_supported_atomic_memory_schema, ) +from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec from powercontext.builtin.persistence.experience_index import ensure_artifact_head_searchable_text +from powercontext.builtin.persistence.migrations.deployment import directory_readiness from powercontext.builtin.persistence.oceanbase.atomic_memory_index import OceanBaseAtomicMemoryIndex from powercontext.builtin.persistence.oceanbase.experience_index import OceanBaseExperienceFTSIndex from powercontext.builtin.persistence.oceanbase.profile import OceanBaseConfig, OceanBaseProfile @@ -992,6 +996,14 @@ async def open_builtin_contexts( database = config.database configured_token_estimator = character_token_estimator() if token_estimator is None else token_estimator if isinstance(database, SQLiteConfig): + directory_state = None if _topic_memory_worker else await asyncio.to_thread(directory_readiness, database) + directory = ( + None + if directory_state is None + else AtomicMemoryDirectoryRepository( + SignedCursorCodec(secret=cursor_secret), available=directory_state and embedding_model is None + ) + ) experience_index = SQLiteExperienceFTSIndex() topic_indexes: list[TopicMemoryIndex] = [SQLiteTopicMemoryFTSIndex()] if embedding_model is not None: @@ -1027,6 +1039,7 @@ async def open_builtin_contexts( database=profile.database, topic_memory_index=topic_index, atomic_memory_index=atomic_index, + atomic_memory_directory=directory, atomic_memory_preview_signing_secret=None if config.runtime.atomic_memory_preview_signing_secret is None else config.runtime.atomic_memory_preview_signing_secret.get_secret_value().encode("utf-8"), diff --git a/src/powercontext/builtin/runtime/relational.py b/src/powercontext/builtin/runtime/relational.py index 6e0ab08457..40de398734 100644 --- a/src/powercontext/builtin/runtime/relational.py +++ b/src/powercontext/builtin/runtime/relational.py @@ -126,6 +126,7 @@ ) from powercontext.builtin.persistence.artifact_readers import TopicMemoryArtifactListReader from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.atomic_memory_directory import AtomicMemoryDirectoryRepository from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndex from powercontext.builtin.persistence.candidates import CandidateRepository from powercontext.builtin.persistence.connectors import ConnectorCheckpointRepository @@ -461,6 +462,7 @@ def __init__( database: AsyncDatabase, topic_memory_index: TopicMemoryIndex | None = None, atomic_memory_index: AtomicMemoryIndex | None = None, + atomic_memory_directory: AtomicMemoryDirectoryRepository | None = None, atomic_memory_preview_signing_secret: bytes | None = None, atomic_memory_preview_signing_key_id: str = "atomic-memory-v1", atomic_memory_preview_ttl_seconds: int = 300, @@ -503,6 +505,7 @@ def __init__( artifact_repository = ArtifactRepository( (Handoff, AtomicMemory, Experience, Skill, Profile, Prompt, TopicMemory), sources=source_repository, + atomic_memory_directory=atomic_memory_directory, ) if atomic_memory_index is None: if database.engine.dialect.name == "sqlite": @@ -546,7 +549,7 @@ def __init__( self.repositories = _Repositories( sources=source_repository, artifacts=artifact_repository, - governance=ArtifactGovernanceRepository(), + governance=ArtifactGovernanceRepository(atomic_memory_directory=atomic_memory_directory), candidates=CandidateRepository({ Experience.family: ExperienceContent, Skill.family: SkillContent, diff --git a/src/powercontext/client/client.py b/src/powercontext/client/client.py index 0f2fa8b5ab..f68c0c766c 100644 --- a/src/powercontext/client/client.py +++ b/src/powercontext/client/client.py @@ -934,6 +934,12 @@ async def search_atomic_memory( ) -> atomic_models.SearchAtomicMemoryResponse: return await self._request(atomic_operations.SEARCH_ATOMIC_MEMORY, request) + async def query_atomic_memory_directory( + self, request: atomic_models.QueryAtomicMemoryDirectoryRequest + ) -> atomic_models.QueryAtomicMemoryDirectoryResponse: + """Read compact exact references; restart pagination when the directory changes.""" + return await self._request(atomic_operations.QUERY_ATOMIC_MEMORY_DIRECTORY, request) + async def merge_atomic_memories( self, request: atomic_models.MergeAtomicMemoryRequest ) -> atomic_models.AtomicMemoryMutationResponse: diff --git a/src/powercontext/http/__init__.py b/src/powercontext/http/__init__.py index 11a64d8793..ed875f7312 100644 --- a/src/powercontext/http/__init__.py +++ b/src/powercontext/http/__init__.py @@ -69,6 +69,7 @@ ArtifactTagSet, ArtifactTagTarget, AssignableSubjectType, + AtomicMemoryDirectoryItem, AtomicMemoryInput, AtomicMemoryLifecycleRequest, AtomicMemoryMutationResponse, @@ -301,6 +302,8 @@ PublishRemoteSkillRequest, PutProfilePolicyRequest, QueryArtifactTagsRequest, + QueryAtomicMemoryDirectoryRequest, + QueryAtomicMemoryDirectoryResponse, ReadinessResponse, ReadinessStatus, RecallTokenDay, @@ -498,6 +501,7 @@ "ArtifactTagSet", "ArtifactTagTarget", "AssignableSubjectType", + "AtomicMemoryDirectoryItem", "AtomicMemoryInput", "AtomicMemoryLifecycleRequest", "AtomicMemoryMutationResponse", @@ -730,6 +734,8 @@ "PublishRemoteSkillRequest", "PutProfilePolicyRequest", "QueryArtifactTagsRequest", + "QueryAtomicMemoryDirectoryRequest", + "QueryAtomicMemoryDirectoryResponse", "ReadinessResponse", "ReadinessStatus", "RecallTokenDay", diff --git a/src/powercontext/http/_generated/models.py b/src/powercontext/http/_generated/models.py index 503de9b187..2b81bbf3b6 100644 --- a/src/powercontext/http/_generated/models.py +++ b/src/powercontext/http/_generated/models.py @@ -3185,6 +3185,39 @@ class AccessAuditPage(BaseModel): next_cursor: Annotated[StrictStr | None, Field(max_length=2048)] +class QueryAtomicMemoryDirectoryRequest(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1, pattern=".*\\S.*")] + states: Annotated[list[AtomicMemoryState], Field(max_length=4, min_length=1)] = [AtomicMemoryState.ACTIVE] + kind: Annotated[StrictStr | None, Field(max_length=128, min_length=1, pattern=".*\\S.*")] = None + tags: Annotated[list[Tag2], Field(max_length=16, validate_default=True)] = [] + tag_match: TagMatch = TagMatch.ALL + limit: Annotated[StrictInt, Field(ge=1, le=100)] = 50 + cursor: Annotated[StrictStr | None, Field(max_length=4096, min_length=1)] = None + + +class AtomicMemoryDirectoryItem(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + artifact: ArtifactReference + kind: Annotated[StrictStr, Field(max_length=128, min_length=1, pattern=".*\\S.*")] + state: AtomicMemoryState + state_version: Annotated[StrictInt, Field(ge=0)] + merged_into_id: Annotated[StrictStr | None, Field(max_length=128, min_length=1)] + + +class QueryAtomicMemoryDirectoryResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + scope_id: Annotated[StrictStr, Field(max_length=256, min_length=1)] + items: Annotated[list[AtomicMemoryDirectoryItem], Field(max_length=100)] + next_cursor: Annotated[StrictStr | None, Field(max_length=4096, min_length=1)] + + class AtomicMemoryWriteContent(BaseModel): model_config = ConfigDict( extra="forbid", diff --git a/src/powercontext/http/_generated/operations.py b/src/powercontext/http/_generated/operations.py index f095d0849c..5ee9e4b3e3 100644 --- a/src/powercontext/http/_generated/operations.py +++ b/src/powercontext/http/_generated/operations.py @@ -133,6 +133,8 @@ PublishRemoteSkillRequest, PutProfilePolicyRequest, QueryArtifactTagsRequest, + QueryAtomicMemoryDirectoryRequest, + QueryAtomicMemoryDirectoryResponse, ReadinessResponse, ReconcileRemoteSkillsRequest, ReconcileRemoteSkillsResponse, @@ -235,6 +237,33 @@ class AccessRequirement(BaseModel): resolver: str +QUERY_ATOMIC_MEMORY_DIRECTORY = Operation[QueryAtomicMemoryDirectoryRequest, QueryAtomicMemoryDirectoryResponse]( + method="POST", + path="/v1/atomic-memory/directory/query", + operation_id="query_atomic_memory_directory", + request_type=QueryAtomicMemoryDirectoryRequest, + request_location="body", + path_parameters=(), + response_type=QueryAtomicMemoryDirectoryResponse, + success_status=200, + summary="Query compact current Atomic Memory references", + tags=("atomic-memory",), + scope_mode="current", + responses={ + 200: {"description": "Compact current directory page."}, + 400: {"description": "The cursor is invalid or its query, identity, policy or directory generation changed."}, + 401: {"$ref": "#/components/responses/Unauthorized"}, + 403: {"$ref": "#/components/responses/Forbidden"}, + 404: {"$ref": "#/components/responses/NotFound"}, + 410: {"description": "The cursor expired; restart from the first page."}, + 413: {"description": "A compact item exceeds the response budget; no partial page is returned."}, + 422: {"$ref": "#/components/responses/InvalidRequest"}, + 500: {"$ref": "#/components/responses/InternalError"}, + 503: {"$ref": "#/components/responses/Unavailable"}, + }, + access=AccessRequirement(action="scope.read", resource="scope", scope_id_field="scope_id", resolver="request"), +) + LIST_ATOMIC_MEMORIES = Operation[ListAtomicMemoryRequest, ListAtomicMemoryResponse]( method="POST", path="/v1/atomic-memory/list", diff --git a/src/powercontext/http/_generated/schema.py b/src/powercontext/http/_generated/schema.py index 8976d14280..95175ef6ac 100644 --- a/src/powercontext/http/_generated/schema.py +++ b/src/powercontext/http/_generated/schema.py @@ -10,6 +10,94 @@ "version": "1.3.0", }, "paths": { + "/v1/atomic-memory/directory/query": { + "post": { + "tags": ["atomic-memory"], + "summary": "Query compact current Atomic Memory references", + "description": "Returns current metadata " + "and exact revisions " + "without loading Memory " + "text or history. Any " + "directory or tag change " + "invalidates " + "continuation; restart " + "from the first page on " + "invalid_cursor or " + "cursor_expired. Requires " + "verified directory " + "migration and writer " + "composition, and a " + "supported local " + "authorization " + "configuration. " + "Production enablement is " + "unavailable until the " + "unified migration bundle " + "manages the complete " + "target. Unavailable " + "codes include " + "artifact_owner_pending, " + "atomic_memory_directory_authorization_unavailable " + "and " + "atomic_memory_directory_unavailable; " + "do not treat failure as " + "an empty page.", + "operationId": "query_atomic_memory_directory", + "requestBody": { + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/QueryAtomicMemoryDirectoryRequest"} + } + }, + "required": True, + }, + "responses": { + "200": { + "description": "Compact current directory page.", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/QueryAtomicMemoryDirectoryResponse"} + } + }, + }, + "400": { + "description": "The " + "cursor " + "is " + "invalid " + "or " + "its " + "query, " + "identity, " + "policy " + "or " + "directory " + "generation " + "changed.", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, + }, + "401": {"$ref": "#/components/responses/Unauthorized"}, + "403": {"$ref": "#/components/responses/Forbidden"}, + "404": {"$ref": "#/components/responses/NotFound"}, + "410": { + "description": "The cursor expired; restart from the first page.", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, + }, + "413": { + "description": "A compact item exceeds the response budget; no partial page is returned.", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}, + }, + "422": {"$ref": "#/components/responses/InvalidRequest"}, + "500": {"$ref": "#/components/responses/InternalError"}, + "503": {"$ref": "#/components/responses/Unavailable"}, + }, + "x-powercontext-scope-mode": "current", + "x-powercontext-access": { + "action": "scope.read", + "resource": {"type": "scope", "scope-id-from": "scope_id"}, + }, + } + }, "/v1/atomic-memory/list": { "post": { "tags": ["atomic-memory"], @@ -11104,6 +11192,64 @@ "type": "object", "required": ["items", "next_cursor"], }, + "QueryAtomicMemoryDirectoryRequest": { + "properties": { + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1, "pattern": ".*\\S.*"}, + "states": { + "items": {"$ref": "#/components/schemas/AtomicMemoryState"}, + "type": "array", + "maxItems": 4, + "minItems": 1, + "uniqueItems": True, + "default": ["active"], + }, + "kind": { + "type": "string", + "maxLength": 128, + "minLength": 1, + "pattern": ".*\\S.*", + "nullable": True, + }, + "tags": { + "items": {"type": "string", "maxLength": 64, "minLength": 1}, + "type": "array", + "maxItems": 16, + "default": [], + }, + "tag_match": {"$ref": "#/components/schemas/TagMatch", "default": "all"}, + "limit": {"type": "integer", "maximum": 100.0, "minimum": 1.0, "default": 50}, + "cursor": {"type": "string", "maxLength": 4096, "minLength": 1, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["scope_id"], + }, + "AtomicMemoryDirectoryItem": { + "properties": { + "artifact": {"$ref": "#/components/schemas/ArtifactReference"}, + "kind": {"type": "string", "maxLength": 128, "minLength": 1, "pattern": ".*\\S.*"}, + "state": {"$ref": "#/components/schemas/AtomicMemoryState"}, + "state_version": {"type": "integer", "minimum": 0.0}, + "merged_into_id": {"type": "string", "maxLength": 128, "minLength": 1, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["artifact", "kind", "state", "state_version", "merged_into_id"], + }, + "QueryAtomicMemoryDirectoryResponse": { + "properties": { + "scope_id": {"type": "string", "maxLength": 256, "minLength": 1}, + "items": { + "items": {"$ref": "#/components/schemas/AtomicMemoryDirectoryItem"}, + "type": "array", + "maxItems": 100, + }, + "next_cursor": {"type": "string", "maxLength": 4096, "minLength": 1, "nullable": True}, + }, + "additionalProperties": False, + "type": "object", + "required": ["scope_id", "items", "next_cursor"], + }, }, "responses": { "BadRequest": { diff --git a/src/powercontext/server/app.py b/src/powercontext/server/app.py index 7bb45d97be..b959821bb1 100644 --- a/src/powercontext/server/app.py +++ b/src/powercontext/server/app.py @@ -153,6 +153,11 @@ ArtifactLifecycleState, InvalidArtifactLifecycleError, ) +from powercontext.builtin.persistence.atomic_memory_directory import ( + AtomicMemoryDirectoryAuthorizationUnavailableError, + AtomicMemoryDirectoryItemTooLargeError, + AtomicMemoryDirectoryUnavailableError, +) from powercontext.builtin.persistence.atomic_memory_identity import legacy_entry_artifact_id from powercontext.builtin.persistence.atomic_memory_index import AtomicMemoryIndexError from powercontext.builtin.persistence.errors import ( @@ -3208,6 +3213,8 @@ def _atomic_execution_context(request: Request, operation: str) -> ArtifactSearc if access is None: if getattr(application, "atomic_memory", None) is None: raise BaseOperationNotSupportedError("artifact_family", "atomic-memory", "runtime application") + if operation == "query_atomic_memory_directory": + return ArtifactSearchExecutionContext(trusted_local=True) return None return ArtifactSearchExecutionContext( principal=_require_principal(), access=access, audit=_access_audit_context(operation) @@ -5266,6 +5273,9 @@ def _map_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None]: access_error = _map_access_error(error) if access_error is not None: return access_error + directory_error = _map_atomic_directory_error(error) + if directory_error is not None: + return directory_error if isinstance(error, (DreamError, EvidenceResolutionError)): statuses = { "idempotency_conflict": 409, @@ -5456,6 +5466,31 @@ def _map_access_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | return None +def _map_atomic_directory_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None] | None: + if isinstance(error, AtomicMemoryDirectoryAuthorizationUnavailableError): + return ( + status.HTTP_503_SERVICE_UNAVAILABLE, + "atomic_memory_directory_authorization_unavailable", + "The access configuration does not support safe directory queries.", + None, + ) + if isinstance(error, AtomicMemoryDirectoryUnavailableError): + return ( + status.HTTP_503_SERVICE_UNAVAILABLE, + "atomic_memory_directory_unavailable", + "The Atomic Memory directory is not ready.", + None, + ) + if isinstance(error, AtomicMemoryDirectoryItemTooLargeError): + return ( + status.HTTP_413_CONTENT_TOO_LARGE, + "atomic_memory_directory_item_too_large", + "An Atomic Memory directory item exceeds the response budget.", + None, + ) + return None + + def _map_base_access_error(error: Exception) -> tuple[int, str, str, dict[str, Any] | None] | None: if isinstance(error, SourceNotEligibleError): return ( diff --git a/src/powercontext/server/atomic_memory.py b/src/powercontext/server/atomic_memory.py index 6068a762a9..460dc1a3b0 100644 --- a/src/powercontext/server/atomic_memory.py +++ b/src/powercontext/server/atomic_memory.py @@ -42,7 +42,10 @@ def _tag_filter(request): if request.tag_match is not None: raise InvalidBaseAccessRequestError("tag_match", "requires tags") return None - return TagFilter(tags=tuple(request.tags), match="all" if request.tag_match is None else request.tag_match.value) + return TagFilter( + tags=tuple(tag.root for tag in request.tags), + match="all" if request.tag_match is None else request.tag_match.value, + ) def _scoped(application, scope_id): @@ -98,6 +101,21 @@ async def list_atomic_memories( items=[record_response(item) for item in result.items], next_cursor=result.next_cursor ) + async def query_atomic_memory_directory( + request: models.QueryAtomicMemoryDirectoryRequest, + http_request: Request, + application: Annotated[Any, Depends(application_dependency)], + ) -> models.QueryAtomicMemoryDirectoryResponse: + result = await _scoped(application, request.scope_id).query_directory( + states=tuple(state.value for state in request.states), + kind=request.kind, + tag_filter=None if not request.tags else _tag_filter(request), + limit=request.limit, + cursor=request.cursor, + context=execution_context(http_request, "query_atomic_memory_directory"), + ) + return models.QueryAtomicMemoryDirectoryResponse.model_validate(result.model_dump(mode="json")) + async def search_atomic_memory( request: models.SearchAtomicMemoryRequest, http_request: Request, @@ -224,6 +242,7 @@ async def restore_atomic_memory( for operation, endpoint in ( (operations.GET_ATOMIC_MEMORY_STATE, get_atomic_memory_state), (operations.LIST_ATOMIC_MEMORIES, list_atomic_memories), + (operations.QUERY_ATOMIC_MEMORY_DIRECTORY, query_atomic_memory_directory), (operations.SEARCH_ATOMIC_MEMORY, search_atomic_memory), (operations.MERGE_ATOMIC_MEMORIES, merge_atomic_memories), (operations.CHANGE_ATOMIC_MEMORY_LIFECYCLE, change_atomic_memory_lifecycle), diff --git a/src/powercontext/server/authz/casbin.py b/src/powercontext/server/authz/casbin.py index 9bc4b8f9f8..8b35cfb66f 100644 --- a/src/powercontext/server/authz/casbin.py +++ b/src/powercontext/server/authz/casbin.py @@ -93,6 +93,7 @@ async def check_batch(self, requests: Sequence[AccessRequest], /) -> tuple[Acces artifact_resources=tuple( request.resource for request in requests if request.resource.type is AccessResourceType.ARTIFACT ), + decision_resources=tuple(request.resource for request in requests), ) revision = contextual_policy_revision(state.policy_revision, requests[0].context.subject_groups) bindings = state.bindings diff --git a/src/powercontext/server/authz/repository.py b/src/powercontext/server/authz/repository.py index 85013c091b..20c945a296 100644 --- a/src/powercontext/server/authz/repository.py +++ b/src/powercontext/server/authz/repository.py @@ -32,7 +32,10 @@ Table, Text, UniqueConstraint, + and_, delete, + exists, + func, insert, or_, select, @@ -42,7 +45,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from powercontext.builtin.persistence.database import AsyncDatabase -from powercontext.builtin.persistence.tables import identity_string +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, identity_string from powercontext.limits import ( MAX_ARTIFACT_FAMILY_LENGTH, MAX_ARTIFACT_ID_LENGTH, @@ -611,6 +614,55 @@ def with_connection(self, connection: AsyncConnection) -> RelationalAccessReposi raise AccessUnavailableError("transactional_relationships_unavailable") return RelationalAccessRepository(self._database, connection=connection) + async def require_scope_content_ready(self, scope_id: str) -> None: + """Fail closed for any missing formal owner without returning the Scope catalog. + + Match the collection catalog's exclusions: legacy Memory is outside the + runtime, and Topic Memory uses Scope ownership. No lifecycle or page + filter may hide an unowned identity elsewhere in the Scope. + """ + + heads = ARTIFACT_HEADS_TABLE + owners = ACCESS_OWNERS_TABLE + formal_owner = exists( + select(1).where( + owners.c.owner_kind == "artifact", + owners.c.object_key_hash + == func.pc_access_artifact_key_hash(heads.c.scope_id, heads.c.family, heads.c.artifact_id), + owners.c.scope_id == heads.c.scope_id, + owners.c.family == heads.c.family, + owners.c.artifact_id == heads.c.artifact_id, + owners.c.selector_type.is_(None), + owners.c.selector_entry_id.is_(None), + ) + ) + async with self._database.connection(self._bound_connection) as connection: + if connection.dialect.name != "sqlite" or connection.dialect.driver != "aiosqlite": + raise AccessUnavailableError("safe_resource_filtering_unavailable") + # Hash the exact canonical ResourceRef through the existing owner + # key authority. SQLite has no native SHA-256. This connection-local + # function enables primary-key probes and does not write schema/data. + await connection.run_sync( + lambda sync: sync.connection.run_async( + lambda driver: driver.create_function( + "pc_access_artifact_key_hash", + 3, + lambda scope, family, identity: _digest( + ResourceRef.artifact(scope, family=family, artifact_id=identity).key + ), + deterministic=True, + ) + ) + ) + missing = await connection.scalar( + select(1) + .select_from(heads) + .where(heads.c.scope_id == scope_id, heads.c.family.not_in(("memory", "topic-memory")), ~formal_owner) + .limit(1) + ) + if missing is not None: + raise AccessUnavailableError("artifact_owner_pending") + async def decision_snapshot( self, subjects: Sequence[AccessSubjectRef], @@ -618,6 +670,21 @@ async def decision_snapshot( now: datetime, artifact_resources: Sequence[ResourceRef] = (), owned_by: PrincipalRef | None = None, + ) -> DecisionState: + """Preserve the public snapshot SPI, including repository overrides.""" + + return await self._decision_snapshot( + subjects, now=now, artifact_resources=artifact_resources, owned_by=owned_by + ) + + async def _decision_snapshot( + self, + subjects: Sequence[AccessSubjectRef], + *, + now: datetime, + artifact_resources: Sequence[ResourceRef] = (), + owned_by: PrincipalRef | None = None, + decision_resources: Sequence[ResourceRef] | None = None, ) -> DecisionState: """Read revision, active bindings and ownership from one snapshot. @@ -642,24 +709,32 @@ async def decision_snapshot( await self._pin_read_snapshot(connection) revision = await self._read_policy_revision(connection) if subjects: - binding_rows = ( - ( - await connection.execute( - select(ACCESS_BINDINGS_TABLE).where( - or_( - *( - (ACCESS_BINDINGS_TABLE.c.subject_type == subject.type) - & (ACCESS_BINDINGS_TABLE.c.subject_id == subject.id) - for subject in subjects - ) - ), - ACCESS_BINDINGS_TABLE.c.state == AccessBindingState.ACTIVE.value, - ) + statement = select(ACCESS_BINDINGS_TABLE).where( + or_( + *( + (ACCESS_BINDINGS_TABLE.c.subject_type == subject.type) + & (ACCESS_BINDINGS_TABLE.c.subject_id == subject.id) + for subject in subjects ) - ) - .mappings() - .all() + ), + ACCESS_BINDINGS_TABLE.c.state == AccessBindingState.ACTIVE.value, ) + if decision_resources is not None: + # A point check needs only exact resources and their Scope + # parents. Keep Server rows for existing inheritance rules. + resources = {resource.key: resource for resource in decision_resources} + resources.update({ + parent.key: parent + for resource in decision_resources + if (parent := resource.parent_scope) is not None + }) + statement = statement.where( + or_( + ACCESS_BINDINGS_TABLE.c.resource_type == "server", + *(and_(*_resource_predicates(ACCESS_BINDINGS_TABLE, item)) for item in resources.values()), + ) + ) + binding_rows = (await connection.execute(statement)).mappings().all() if artifact_resources: owner_rows = ( ( diff --git a/src/powercontext/server/authz/service.py b/src/powercontext/server/authz/service.py index c1078a0ec3..42698af4c7 100644 --- a/src/powercontext/server/authz/service.py +++ b/src/powercontext/server/authz/service.py @@ -374,13 +374,14 @@ async def decision_snapshot( _SNAPSHOT_RETRY_LIMIT = 3 -async def read_decision_state( +async def read_decision_state( # noqa: C901 repository: AccessRepository, subjects: Sequence[AccessSubjectRef], *, now: datetime, artifact_resources: Sequence[ResourceRef] = (), owned_by: PrincipalRef | None = None, + decision_resources: Sequence[ResourceRef] | None = None, ) -> DecisionState: """Read decision inputs from one snapshot when the repository offers one. @@ -399,6 +400,19 @@ async def read_decision_state( last_error: AccessUnavailableError | None = None for _ in range(_SNAPSHOT_RETRY_LIMIT): try: + if decision_resources is not None: + from powercontext.server.authz.repository import RelationalAccessRepository + + # Only the canonical adapter promises point-resource + # selection. External/subclass snapshot contracts are kept. + if type(repository) is RelationalAccessRepository: + return await repository._decision_snapshot( + subjects, + now=now, + artifact_resources=artifact_resources, + owned_by=owned_by, + decision_resources=decision_resources, + ) return await snapshot_reader( subjects, now=now, artifact_resources=artifact_resources, owned_by=owned_by ) @@ -505,6 +519,7 @@ async def check_batch( artifact_resources=tuple( request.resource for request in requests if request.resource.type is AccessResourceType.ARTIFACT ), + decision_resources=tuple(request.resource for request in requests), ) revision = contextual_policy_revision(state.policy_revision, requests[0].context.subject_groups) bindings = state.bindings @@ -660,6 +675,54 @@ async def require_scope_read(self, principal, scope_id: str, *, connection, cont access._deferred_decisions = self._deferred_decisions return await access.require(principal, AccessAction.SCOPE_READ, ResourceRef.scope(scope_id), context=context) + async def require_scope_directory_read( + self, principal, scope_id: str, *, connection, context: AccessAuditContext + ) -> str: + """Qualify a Scope-wide SQL directory without replacing custom policy or audit. + + A successful Scope read permits all catalog rows, so no per-object + authorization scan is necessary. Formal ownership remains a separate, + Scope-wide fail-closed precondition in the same database snapshot. + Return the policy/identity context digest owned by this authority. + """ + + decision = await self.require_scope_read(principal, scope_id, connection=connection, context=context) + from powercontext.server.authz.repository import RelationalAccessRepository + + provider = self.provider + canonical = type(provider) is BuiltinAuthorizationProvider + if not canonical and type(provider).__module__ == "powercontext.server.authz.casbin": + from powercontext.server.authz.casbin import CasbinAuthorizationProvider + + canonical = type(provider) is CasbinAuthorizationProvider + policy_repository = getattr(provider, "_repository", None) + relationships = self._relationship_reader() + if ( + not self.provider_capabilities.safe_resource_filtering + or not canonical + or type(policy_repository) is not RelationalAccessRepository + or type(relationships) is not RelationalAccessRepository + or decision.policy_revision is None + ): + raise AccessUnavailableError("safe_resource_filtering_unavailable") + # Verify both authorities belong to the caller's database. Different + # audit storage remains supported through the original access service. + policy_repository.with_connection(connection) + await relationships.with_connection(connection).require_scope_content_ready(scope_id) + return hashlib.sha256( + json.dumps( + { + "provider": f"{type(provider).__module__}.{type(provider).__qualname__}", + "deployment": self.deployment_id, + "revision": decision.policy_revision, + "groups": sorted({group.id for group in context.subject_groups}), + }, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ).encode() + ).hexdigest() + async def bootstrap_subject_scope(self, connection, principal, scope_id, *, context): """Grant only a newly created ordinary Scope, in the Source transaction.""" from powercontext.server.authz.repository import RelationalAccessRepository diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index d813cff2f9..22672f042a 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -37,6 +37,12 @@ from powercontext.service.model import ServiceError BackupPolicy = Literal["auto", "manual", "skip"] +MigrationTarget = Annotated[ + Literal["atomic-memory-directory"] | None, + typer.Option( + help="Select the registered current SQLite directory target; default is the four-table acceptance target." + ), +] EnvFile = Annotated[Path | None, typer.Option(help="Read the same environment file as the deployed Server.")] EvidenceDir = Annotated[ Path | None, @@ -72,18 +78,24 @@ def main() -> None: @app.command() def status( - env_file: EnvFile = None, evidence_dir: EvidenceDir = None, lock_coordination: LockCoordination = None + env_file: EnvFile = None, + evidence_dir: EvidenceDir = None, + lock_coordination: LockCoordination = None, + target: MigrationTarget = None, ) -> None: """Inspect the schema version without creating a database or control tables.""" - _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=False) + _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=False, target=target) @app.command() def verify( - env_file: EnvFile = None, evidence_dir: EvidenceDir = None, lock_coordination: LockCoordination = None + env_file: EnvFile = None, + evidence_dir: EvidenceDir = None, + lock_coordination: LockCoordination = None, + target: MigrationTarget = None, ) -> None: """Verify database readiness; this does not assert Server or cluster readiness.""" - _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=True) + _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=True, target=target) @app.command() @@ -94,10 +106,13 @@ def plan( backup: Annotated[BackupPolicy, typer.Option(help="Bind the selected backup policy to the plan.")] = "auto", manage_service: ManageService = False, shared_database: SharedDatabase = False, + target: MigrationTarget = None, ) -> None: """Preview the target, resource digests, backup policy, and maintenance scope.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination) + runner = deployment_runner( + settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination, target=target + ) service = _service_scope(_environment_file(env_file)) if manage_service else None reviewed = runner.plan(backup_policy=backup, service=service, shared_database=shared_database) _write_payload(reviewed) @@ -137,6 +152,7 @@ def apply( backup_ref: Annotated[str | None, typer.Option(help="Optional reference for the operator's manual backup.")] = None, manage_service: ManageService = False, shared_database: SharedDatabase = False, + target: MigrationTarget = None, ) -> None: """Review once, then stop writes, back up, migrate, verify, and optionally switch a local service.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: @@ -145,6 +161,7 @@ def apply( evidence_dir=evidence_dir, lock_coordination=lock_coordination, acknowledge_previous_run=acknowledge_previous_run, + target=target, ) policy: BackupPolicy = backup or "auto" initial = runner.plan(backup_policy=policy, shared_database=shared_database) @@ -199,9 +216,12 @@ def _inspect( evidence_dir: Path | None, lock_coordination: Literal["single-host"] | None, verify: bool, + target: Literal["atomic-memory-directory"] | None, ) -> None: with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination) + runner = deployment_runner( + settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination, target=target + ) result = runner.verify() if verify else runner.plan() _write_payload(result) diff --git a/src/powercontext/server/mcp.py b/src/powercontext/server/mcp.py index 0cf9ca8955..e6dab8aa73 100644 --- a/src/powercontext/server/mcp.py +++ b/src/powercontext/server/mcp.py @@ -112,7 +112,10 @@ Ordinary coding needs no routine Memory calls. Use sufficient current context when continuing work. For an explicit memory search (search my memories / 搜索记忆), call search_memory with a focused query, mode auto, and at most eight hits. search_memory and list_memory_entries return current Atomic Memory records with real ArtifactRef and state_version. -Use list_atomic_memories for explicit state filters and pagination. get_memory_entry accepts only a legacy logical +Use list_atomic_memories when full records are needed. query_atomic_memory_directory browses compact exact references +and states without content. Keep its filters and limit unchanged on continuation; if a cursor is invalid or expired, +restart at page one. Read directory content with get_artifact_revision using the returned exact reference. +get_memory_entry accepts only a legacy logical target; it returns current Atomic Memory and does not follow merged_into_id automatically. Historical citations are unsupported. For new identities use get_artifact for the current content and get_artifact_revision for exact historical content. @@ -152,6 +155,7 @@ """ _MCP_OPERATION_IDS = frozenset({ "get_atomic_memory_state", + "query_atomic_memory_directory", "list_atomic_memories", "search_atomic_memory", "merge_atomic_memories", @@ -208,6 +212,7 @@ }) _MCP_READ_ONLY_OPERATION_IDS = frozenset({ "get_atomic_memory_state", + "query_atomic_memory_directory", "list_atomic_memories", "search_atomic_memory", "preview_atomic_memory_restoration", diff --git a/tests/builtin/persistence/test_atomic_memory_directory.py b/tests/builtin/persistence/test_atomic_memory_directory.py new file mode 100644 index 0000000000..5d38825e70 --- /dev/null +++ b/tests/builtin/persistence/test_atomic_memory_directory.py @@ -0,0 +1,860 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compact current-directory storage, maintenance and shared authorization contracts.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from hashlib import sha256 +from pathlib import Path + +import pytest +from pydantic import ValidationError +from sqlalchemy import BigInteger, Column, MetaData, event, insert, select, true, update + +from powercontext.artifacts import ArtifactAddress +from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryContent, AtomicMemoryDraft +from powercontext.builtin.artifacts.atomic_memory.service import AtomicMemoryMergeAdapter +from powercontext.builtin.artifacts.merge import ArtifactMergeService +from powercontext.builtin.persistence import atomic_memory_directory as directory +from powercontext.builtin.persistence.artifact_governance import ArtifactGovernanceRepository, ArtifactLifecycleState +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.atomic_memory import AtomicMemoryStateRepository +from powercontext.builtin.persistence.atomic_memory_directory import ( + AtomicMemoryDirectoryItemTooLargeError, + AtomicMemoryDirectoryQuery, + AtomicMemoryDirectoryRepository, + AtomicMemoryDirectoryUnavailableError, +) +from powercontext.builtin.persistence.atomic_memory_index import PreparedAtomicMemoryProjection +from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.sources import SourceRepository +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import ( + ARTIFACT_HEADS_TABLE, + ARTIFACT_TAGS_TABLE, + ARTIFACTS_TABLE, + SCOPE_TABLES, + SHARED_TABLES, + identity_string, +) +from powercontext.builtin.persistence.tags import RelationalTagService +from powercontext.builtin.records import CursorExpiredError, InvalidCursorError +from powercontext.builtin.runtime.artifact_merge import ArtifactMergeApplication, merge_artifact_tags +from powercontext.builtin.sources.content import CONTENT_SOURCE_ADAPTER +from powercontext.builtin.tags import ArtifactTagTarget, TagFilter + + +@asynccontextmanager +async def directory_profile(path: Path | None = None, *, extra_tables=()): + # A disposable target-schema fixture, not a startup/migration implementation. + metadata = MetaData() + for shared in (*SHARED_TABLES, *SCOPE_TABLES, ARTIFACT_TAGS_TABLE, *extra_tables): + shared.to_metadata(metadata) + scopes = metadata.tables["pc_scopes"] + scopes.append_column(Column("atomic_memory_directory_generation", BigInteger, nullable=False, server_default="0")) + heads = metadata.tables["pc_artifact_heads"] + heads.append_column(Column("content_kind", identity_string(128))) + config = SQLiteConfig() if path is None else SQLiteConfig(url=f"sqlite+aiosqlite:///{path}") + async with SQLiteProfile.open(config, tables=tuple(metadata.sorted_tables)) as profile: + async with profile.database.transaction() as connection: + await connection.execute( + insert(scopes).values( + scope_id="scope-a", + title="A", + summary="A", + scope_id_search="scope-a", + title_search="a", + summary_search="a", + version=1, + ) + ) + yield profile, scopes, heads + + +async def publish(connection, reader, identity="a", *, kind="rule", text="Keep this rule."): + return await ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader).create( + connection, "scope-a", identity, AtomicMemoryDraft(content=AtomicMemoryContent(kind=kind, text=text)) + ) + + +async def query(connection, reader, *, eligibility=None, subject="user:a", policy_context="fixture-local", **kwargs): + # true() is an explicit trusted storage fixture; it does not qualify authz support. + return await reader.query( + connection, + AtomicMemoryDirectoryQuery(scope_id="scope-a", **kwargs), + subject=subject, + policy_context=policy_context, + eligibility=true() if eligibility is None else eligibility, + ) + + +def test_exact_revision_directory_completes_and_content_change_requires_restart(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, _, _): + async with profile.database.transaction() as connection: + original = await publish(connection, reader, "a") + await publish(connection, reader, "b") + await publish(connection, reader, "c") + async with profile.database.transaction(consistent_snapshot=True) as connection: + first = await query(connection, reader, limit=2) + last = await query(connection, reader, limit=2, cursor=first.next_cursor) + assert [item.artifact.artifact_id for item in first.items + last.items] == ["a", "b", "c"] + assert first.items[0].artifact == original.as_ref() + assert first.next_cursor is not None and last.next_cursor is None + assert "text" not in first.items[0].model_dump() + async with profile.database.transaction() as connection: + revised = await ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader).revise( + connection, + "scope-a", + original, + AtomicMemoryDraft(content=AtomicMemoryContent(kind="preference", text="Use short answers.")), + ) + assert isinstance(revised, AtomicMemory) + async with profile.database.transaction(consistent_snapshot=True) as connection: + with pytest.raises(InvalidCursorError): + await query(connection, reader, limit=2, cursor=first.next_cursor) + current = await query(connection, reader, kind="preference") + assert current.items[0].artifact.revision == 2 + exact = await ArtifactRepository((AtomicMemory,)).get(connection, "scope-a", original.as_ref()) + assert exact.content.text == original.content.text + + asyncio.run(scenario()) + + +def test_locked_evidence_reads_and_direct_merge_transitions_take_scope_lock_first(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, scopes, _), profile.database.transaction() as connection: + artifact = await publish(connection, reader, "a") + repository = ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader) + governance = ArtifactGovernanceRepository(atomic_memory_directory=reader) + before = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + statements = [] + + def observe(_connection, _cursor, statement, _parameters, _context, _many): + statements.append(statement.lower().lstrip()) + + event.listen(profile.database.engine.sync_engine, "before_cursor_execute", observe) + try: + for read in ( + lambda: repository.latest(connection, "scope-a", "atomic-memory", "a", for_update=True), + lambda: repository.get(connection, "scope-a", artifact.as_ref(), for_update=True), + ): + statements.clear() + assert await read() == artifact + assert statements[0].startswith("update pc_scopes") + current = await governance.get(connection, "scope-a", "atomic-memory", "a") + statements.clear() + await governance.transition_merge( + connection, "scope-a", "atomic-memory", "a", current, ArtifactLifecycleState.ACTIVE + ) + assert statements[0].startswith("update pc_scopes") + finally: + event.remove(profile.database.engine.sync_engine, "before_cursor_execute", observe) + # Locking is not an effective directory mutation, including a no-op merge. + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == before + + asyncio.run(scenario()) + + +def test_lifecycle_kind_tags_and_supplied_eligibility_are_applied_before_limit(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, _, _), profile.database.transaction() as connection: + for identity, kind in (("a", "other"), ("b", "rule"), ("c", "rule"), ("d", "rule")): + await publish(connection, reader, identity, kind=kind) + governance = ArtifactGovernanceRepository(atomic_memory_directory=reader) + current = await governance.get(connection, "scope-a", "atomic-memory", "c") + await governance.transition_merge( + connection, "scope-a", "atomic-memory", "c", current, ArtifactLifecycleState.DEPRECATED, "d" + ) + for identity in ("b", "c"): + await connection.execute( + insert(ARTIFACT_TAGS_TABLE).values( + scope_id="scope-a", + family="atomic-memory", + artifact_id=identity, + target_type="artifact", + target_id=identity, + tag_key="rule", + tag_key_hash=sha256(b"rule").digest(), + tag="Rule", + assigned_at=datetime.now(UTC), + ) + ) + await reader.record_mutation(connection, "scope-a") + selected = await query( + connection, + reader, + states=("active", "merged"), + kind="rule", + limit=1, + tag_filter=TagFilter(tags=("RULE",)), + eligibility=ARTIFACT_HEADS_TABLE.c.artifact_id != "b", + ) + assert [item.artifact.artifact_id for item in selected.items] == ["c"] + assert selected.items[0].state == "merged" + assert selected.items[0].state_version == 1 + assert selected.items[0].merged_into_id == "d" + assert selected.next_cursor is None + + asyncio.run(scenario()) + + +def test_long_normalized_filters_continue_and_context_changes_reject_cursor(): + async def scenario(): + now = [datetime(2026, 10, 10, tzinfo=UTC)] + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture", clock=lambda: now[0])) + labels = tuple(chr(0x4E00 + index) * 64 for index in range(16)) + async with directory_profile() as (profile, _, _), profile.database.transaction() as connection: + for identity in ("a", "b"): + await publish(connection, reader, identity) + await connection.execute( + insert(ARTIFACT_TAGS_TABLE), + [ + { + "scope_id": "scope-a", + "family": "atomic-memory", + "artifact_id": identity, + "target_type": "artifact", + "target_id": identity, + "tag_key": label, + "tag_key_hash": sha256(label.encode()).digest(), + "tag": label, + "assigned_at": datetime.now(UTC), + } + for label in labels + ], + ) + first = await query(connection, reader, limit=1, tag_filter=TagFilter(tags=labels)) + assert first.next_cursor is not None and len(first.next_cursor) <= 4096 + tail = await query( + connection, + reader, + limit=1, + cursor=first.next_cursor, + tag_filter=TagFilter(tags=tuple(reversed(labels))), + ) + assert tail.items[0].artifact.artifact_id == "b" and tail.next_cursor is None + for changes in ({"subject": "user:b"}, {"policy_context": "revoked"}, {"limit": 2}, {"kind": "other"}): + with pytest.raises(InvalidCursorError): + await query( + connection, + reader, + cursor=first.next_cursor, + tag_filter=TagFilter(tags=labels), + **{"limit": 1, **changes}, + ) + with pytest.raises(InvalidCursorError): + await query( + connection, reader, limit=1, tag_filter=TagFilter(tags=labels), cursor="!" + first.next_cursor[1:] + ) + now[0] += timedelta(hours=1) + with pytest.raises(CursorExpiredError): + await query(connection, reader, limit=1, tag_filter=TagFilter(tags=labels), cursor=first.next_cursor) + + asyncio.run(scenario()) + + +def test_atomic_state_noop_preserves_cursor_but_forget_and_restore_invalidate_it(): + async def scenario(): + from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemoryStateValue + + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + states = AtomicMemoryStateRepository(atomic_memory_directory=reader) + async with directory_profile() as (profile, scopes, _), profile.database.transaction() as connection: + await publish(connection, reader, "a") + await publish(connection, reader, "b") + first = await query(connection, reader, limit=1) + generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + original = await states.get(connection, "scope-a", "a") + unchanged = await states.transition(connection, "scope-a", "a", original, AtomicMemoryStateValue.ACTIVE) + assert unchanged == original + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + assert (await query(connection, reader, limit=1, cursor=first.next_cursor)).items[ + 0 + ].artifact.artifact_id == "b" + forgotten = await states.transition(connection, "scope-a", "a", original, AtomicMemoryStateValue.FORGOTTEN) + with pytest.raises(InvalidCursorError): + await query(connection, reader, limit=1, cursor=first.next_cursor) + page = await query(connection, reader, states=("active", "forgotten"), limit=1) + assert page.items[0].state == "forgotten" + await states.transition(connection, "scope-a", "a", forgotten, AtomicMemoryStateValue.ACTIVE) + with pytest.raises(InvalidCursorError): + await query(connection, reader, states=("active", "forgotten"), limit=1, cursor=page.next_cursor) + + asyncio.run(scenario()) + + +def test_tag_replace_noop_preserves_pages_and_display_change_invalidates_unfiltered_pages(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, scopes, _): + artifacts = ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader) + tags = RelationalTagService(profile.database, artifacts) + target = ArtifactTagTarget(family="atomic-memory", artifact_id="a") + async with profile.database.transaction() as connection: + await publish(connection, reader, "a") + await publish(connection, reader, "b") + for identity in ("a", "b"): + current_target = target.model_copy(update={"artifact_id": identity}) + empty = await tags.get("scope-a", current_target) + await tags.replace("scope-a", current_target, ("Rule", "中文"), expected_etag=empty.etag) + async with profile.database.transaction(consistent_snapshot=True) as connection: + plain = await query(connection, reader, limit=1) + filtered = await query(connection, reader, limit=1, tag_filter=TagFilter(tags=("rule",))) + generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + current = await tags.get("scope-a", target) + unchanged = await tags.replace("scope-a", target, ("中文", "Rule"), expected_etag=current.etag) + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + assert (await query(connection, reader, limit=1, cursor=plain.next_cursor)).items + # The normalized membership is identical, but the displayed tag changed. + await tags.replace("scope-a", target, ("RULE", "中文"), expected_etag=unchanged.etag) + async with profile.database.transaction(consistent_snapshot=True) as connection: + for page, filters in ((plain, {}), (filtered, {"tag_filter": TagFilter(tags=("rule",))})): + with pytest.raises(InvalidCursorError): + await query(connection, reader, limit=1, cursor=page.next_cursor, **filters) + assert "RULE" in (await tags.get("scope-a", target)).tags + + asyncio.run(scenario()) + + +class _TrustedStorageSecurity: + """An explicit storage fixture, without production ownership or policy claims.""" + + def subject(self, context): + return context + + async def authorize(self, connection, scope_id, context, action, ref=None): + pass + + async def authorize_sources(self, connection, scope_id, context, sources): + pass + + async def establish_owner(self, connection, scope_id, artifact_id, context): + pass + + +class _StorageProjections: + """Keep search outside this test and inject a failure after a business write.""" + + fail_publication = False + + async def prepare(self, content): + return PreparedAtomicMemoryProjection( + kind=content.kind, + text=content.text, + searchable_text=content.text, + content_hash=sha256(content.text.encode()).hexdigest(), + ) + + def validate_prepared(self, prepared): + pass + + async def publish(self, connection, scope_id, record, prepared): + if self.fail_publication: + raise RuntimeError("projection publication failed") # noqa: TRY003 + + async def remove(self, connection, scope_id, artifact_id): + pass + + +def test_cascade_merge_restore_and_failed_publication_keep_directory_atomic(tmp_path): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile(tmp_path / "merge.sqlite") as (profile, scopes, _): + sources = SourceRepository((CONTENT_SOURCE_ADAPTER,)) + artifacts = ArtifactRepository((AtomicMemory,), sources=sources, atomic_memory_directory=reader) + projections = _StorageProjections() + + async def merge_tags(connection, scope_id, result_id, inputs): + await merge_artifact_tags(connection, scope_id, result_id, inputs, family="atomic-memory") + + service = ArtifactMergeService( + artifacts=artifacts, + adapter=AtomicMemoryMergeAdapter(projections), + security=_TrustedStorageSecurity(), + merge_tags=merge_tags, + ) + memory = ArtifactMergeApplication(profile.database, (service,), default_context="fixture-owner").for_scope( + "scope-a", "atomic-memory" + ) + async with profile.database.transaction() as connection: + for identity in ("a", "b", "d"): + await publish(connection, reader, identity) + async with profile.database.transaction(consistent_snapshot=True) as connection: + initial = await query(connection, reader, limit=1) + first = await memory.merge( + tuple([(await memory.get(identity)).as_read() for identity in ("a", "b")]), + AtomicMemoryContent(kind="combined", text="Keep both rules."), + artifact_id="c", + ) + await memory.merge( + (first.primary.as_read(), (await memory.get("d")).as_read()), + AtomicMemoryContent(kind="summary", text="Keep all three rules."), + artifact_id="e", + ) + async with profile.database.transaction(consistent_snapshot=True) as connection: + with pytest.raises(InvalidCursorError): + await query(connection, reader, limit=1, cursor=initial.next_cursor) + active = await query(connection, reader) + assert [(item.artifact.artifact_id, item.kind) for item in active.items] == [("e", "summary")] + before = await query(connection, reader, states=("active", "merged"), limit=1) + generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + projections.fail_publication = True + with pytest.raises(RuntimeError, match="projection publication failed"): + await memory.restore("b") + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + assert (await query(connection, reader)).items == active.items + assert ( + await query(connection, reader, states=("active", "merged"), limit=1, cursor=before.next_cursor) + ).items + projections.fail_publication = False + restored = await memory.restore("b") + assert restored.undo_merge_results == ("e", "c") + async with profile.database.transaction(consistent_snapshot=True) as connection: + with pytest.raises(InvalidCursorError): + await query(connection, reader, states=("active", "merged"), limit=1, cursor=before.next_cursor) + final = await query(connection, reader, states=("active", "retired")) + assert {item.artifact.artifact_id: (item.state.value, item.kind) for item in final.items} == { + "a": ("active", "rule"), + "b": ("active", "rule"), + "c": ("retired", "combined"), + "d": ("active", "rule"), + "e": ("retired", "summary"), + } + assert all(item.artifact.revision == 2 for item in final.items) + generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + assert not (await memory.restore("a")).changed + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + + asyncio.run(scenario()) + + +def test_copy_exact_maintains_kind_and_overflow_rolls_back_business_publication(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + artifacts = ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader) + async with directory_profile() as (profile, scopes, _): + async with profile.database.transaction() as connection: + original = await publish(connection, reader) + copied = await artifacts.copy_exact( + connection, + "scope-a", + "copy", + ArtifactAddress(scope_id="scope-a", artifact=original.as_ref()), + original, + sha256(original.content.model_dump_json().encode()).hexdigest(), + ) + page = await query(connection, reader) + assert page.items[1].artifact == copied.as_ref() and page.items[1].kind == original.content.kind + await connection.execute(update(scopes).values(atomic_memory_directory_generation=(1 << 63) - 1)) + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + async with profile.database.transaction() as connection: + await publish(connection, reader, "never-committed") + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert {item.artifact.artifact_id for item in (await query(connection, reader)).items} == {"a", "copy"} + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == (1 << 63) - 1 + + asyncio.run(scenario()) + + +def test_other_scopes_and_families_do_not_invalidate_atomic_directory(): + async def scenario(): + from tests.builtin.persistence.contract import Handoff, HandoffContent, HandoffDraft + + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + artifacts = ArtifactRepository((AtomicMemory, Handoff), atomic_memory_directory=reader) + async with directory_profile() as (profile, scopes, _), profile.database.transaction() as connection: + for identity in ("a", "b"): + await publish(connection, reader, identity) + first = await query(connection, reader, limit=1) + await connection.execute( + insert(scopes).values( + scope_id="scope-b", + title="B", + summary="B", + scope_id_search="scope-b", + title_search="b", + summary_search="b", + version=1, + ) + ) + await artifacts.create( + connection, + "scope-b", + "other-memory", + AtomicMemoryDraft(content=AtomicMemoryContent(kind="rule", text="Another Scope.")), + ) + handoff = await artifacts.create( + connection, "scope-a", "handoff", HandoffDraft(content=HandoffContent(summary="A different Family.")) + ) + await artifacts.revise( + connection, "scope-a", handoff, HandoffDraft(content=HandoffContent(summary="A revised handoff.")) + ) + assert (await query(connection, reader, limit=1, cursor=first.next_cursor)).items[ + 0 + ].artifact.artifact_id == "b" + assert ( + await connection.scalar( + select(scopes.c.atomic_memory_directory_generation).where(scopes.c.scope_id == "scope-b") + ) + == 1 + ) + + asyncio.run(scenario()) + + +def test_directory_maintenance_batches_are_readonly_idempotent_and_preserve_exact_history(tmp_path): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile(tmp_path / "maintenance.sqlite") as (profile, scopes, heads): + async with profile.database.transaction() as connection: + originals = [await publish(connection, reader, identity) for identity in ("a", "b", "c")] + page = await query(connection, reader, limit=1) + generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + await connection.execute(update(heads).values(content_kind=None)) + # Plan/verify is read-only even when it detects incomplete metadata. + async with profile.database.transaction(consistent_snapshot=True) as connection: + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await reader.maintain_batch(connection, "scope-a", batch_size=2) + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + assert (await connection.execute(select(heads.c.content_kind))).scalars().all() == [None] * 3 + # Interrupt after one committed batch and restart from the beginning. + async with profile.database.transaction() as connection: + assert await reader.maintain_batch(connection, "scope-a", batch_size=2, repair=True) == ("b", 2) + async with profile.database.transaction() as connection: + assert await reader.maintain_batch(connection, "scope-a", batch_size=2, repair=True) == ("b", 0) + assert await reader.maintain_batch(connection, "scope-a", after="b", batch_size=2, repair=True) == ( + None, + 1, + ) + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + 2 + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert await reader.maintain_batch(connection, "scope-a", batch_size=2) == ("b", 0) + assert await reader.maintain_batch(connection, "scope-a", after="b", batch_size=2) == (None, 0) + with pytest.raises(InvalidCursorError): + await query(connection, reader, limit=1, cursor=page.next_cursor) + for original in originals: + assert ( + await ArtifactRepository((AtomicMemory,)).get(connection, "scope-a", original.as_ref()) + == original + ) + + asyncio.run(scenario()) + + +def test_directory_maintenance_validates_whole_batch_and_caller_can_roll_back_repair(tmp_path): + async def scenario(): + from powercontext.builtin.persistence.errors import InvalidStoredPayloadError + + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile(tmp_path / "repair.sqlite") as (profile, scopes, heads): + async with profile.database.transaction() as connection: + for identity in ("a", "b"): + await publish(connection, reader, identity) + await connection.execute(update(heads).values(content_kind="incorrect")) + await connection.execute( + update(ARTIFACTS_TABLE).where(ARTIFACTS_TABLE.c.artifact_id == "b").values(content=b"broken") + ) + generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) + async with profile.database.transaction() as connection: + with pytest.raises(InvalidStoredPayloadError): + await reader.maintain_batch(connection, "scope-a", repair=True) + assert (await connection.execute(select(heads.c.content_kind))).scalars().all() == ["incorrect"] * 2 + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + async with profile.database.engine.connect() as connection: + transaction = await connection.begin() + assert await reader.maintain_batch(connection, "scope-a", batch_size=1, repair=True) == ("a", 1) + assert connection.in_transaction() and not connection.closed + await transaction.rollback() + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert (await connection.execute(select(heads.c.content_kind))).scalars().all() == ["incorrect"] * 2 + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("size", (200, 1000, 5000)) +def test_scope_owner_ready_uses_one_bounded_result_and_exact_canonical_owner_keys(size, record_property, monkeypatch): + async def scenario(): + from time import perf_counter + + from powercontext.server.authz import AccessUnavailableError, ResourceRef + from powercontext.server.authz.repository import ACCESS_OWNERS_TABLE, ACCESS_TABLES, RelationalAccessRepository + + async with directory_profile(extra_tables=ACCESS_TABLES) as (profile, _, heads): + async with profile.database.transaction() as connection: + ids = tuple(f"记忆-{index:05d}" for index in range(size)) + await connection.execute( + insert(ARTIFACTS_TABLE), + [ + { + "scope_id": "scope-a", + "family": "atomic-memory", + "artifact_id": identity, + "revision": 1, + "content": b"not decoded", + } + for identity in ids + ], + ) + await connection.execute( + insert(heads), + [ + {"scope_id": "scope-a", "family": "atomic-memory", "artifact_id": identity, "revision": 1} + for identity in ids + ], + ) + await connection.execute( + insert(ACCESS_OWNERS_TABLE), + [ + { + "owner_kind": "artifact", + "object_key_hash": sha256( + ResourceRef.artifact( + "scope-a", family="atomic-memory", artifact_id=identity + ).key.encode() + ).hexdigest(), + "scope_id": "scope-a", + "family": "atomic-memory", + "artifact_id": identity, + "owner_type": "user", + "owner_id": "reader", + "established_at": datetime.now(UTC).isoformat(), + "policy_revision": "0", + "idempotency_key": identity, + } + for identity in ids + ], + ) + async with profile.database.transaction(consistent_snapshot=True) as connection: + execute = type(connection).execute + returned_rows = [] + + async def measured(current, statement, *args, **kwargs): + result = await execute(current, statement, *args, **kwargs) + if current is connection and result.returns_rows: + frozen = result.freeze() + returned_rows.append(len(frozen.data)) + return frozen() + return result + + monkeypatch.setattr(type(connection), "execute", measured) + started = perf_counter() + await RelationalAccessRepository(profile.database, connection=connection).require_scope_content_ready( + "scope-a" + ) + record_property("owner_ready_ms", round((perf_counter() - started) * 1000, 3)) + assert returned_rows == [0] + # A coordinate-matching row with the wrong canonical hash is not an owner. + async with profile.database.transaction() as connection: + await connection.execute( + update(ACCESS_OWNERS_TABLE) + .where(ACCESS_OWNERS_TABLE.c.artifact_id == ids[-1]) + .values(object_key_hash="0" * 64) + ) + with pytest.raises(AccessUnavailableError, match="owner"): + await RelationalAccessRepository( + profile.database, connection=connection + ).require_scope_content_ready("scope-a") + + asyncio.run(scenario()) + + +def test_custom_directory_policy_is_rejected_without_replacing_ordinary_scope_read(): + async def scenario(): + from powercontext.server.authz import ( + AccessAuditContext, + AccessBinding, + AccessBindingState, + AccessControlService, + AccessRole, + AccessUnavailableError, + BuiltinAuthorizationProvider, + PrincipalRef, + ResourceRef, + ) + from powercontext.server.authz.repository import ACCESS_TABLES, RelationalAccessRepository + + class CustomPolicy(BuiltinAuthorizationProvider): + pass + + principal = PrincipalRef(type="user", id="reader") + context = AccessAuditContext(transport="test", operation="custom-policy") + async with directory_profile(extra_tables=ACCESS_TABLES) as (profile, _, _): + repository = RelationalAccessRepository(profile.database) + await repository.create_binding( + AccessBinding( + binding_id="scope-reader", + subject=principal, + resource=ResourceRef.scope("scope-a"), + role=AccessRole.SCOPE_VIEWER, + granted_by=principal, + reason="fixture", + created_at=datetime.now(UTC), + expires_at=None, + state=AccessBindingState.ACTIVE, + version=1, + policy_revision="pending", + idempotency_key="fixture", + ) + ) + provider = CustomPolicy(repository) + access = AccessControlService(provider, relationships=repository, audit=repository) + async with ( + access.defer_decision_audit(), + profile.database.transaction(consistent_snapshot=True) as connection, + ): + assert ( + await access.require_scope_read(principal, "scope-a", connection=connection, context=context) + ).allowed + with pytest.raises(AccessUnavailableError) as unavailable: + await access.require_scope_directory_read( + principal, "scope-a", connection=connection, context=context + ) + assert unavailable.value.code == "safe_resource_filtering_unavailable" + assert access.provider is provider + + asyncio.run(scenario()) + + +def test_byte_boundary_preserves_the_next_eligible_item_and_oversized_item_fails(monkeypatch): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, _, _), profile.database.transaction() as connection: + await publish(connection, reader, "a") + await publish(connection, reader, "b") + initial = await query(connection, reader, limit=2) + first_size = len(initial.items[0].model_dump_json().encode()) + 2 + monkeypatch.setattr(directory, "DIRECTORY_ITEM_BYTES", first_size) + first = await query(connection, reader, limit=2) + assert len(first.items) == 1 and first.next_cursor is not None + last = await query(connection, reader, limit=2, cursor=first.next_cursor) + assert [item.artifact.artifact_id for item in first.items + last.items] == ["a", "b"] + assert last.next_cursor is None + monkeypatch.setattr(directory, "DIRECTORY_ITEM_BYTES", first_size - 1) + with pytest.raises(AtomicMemoryDirectoryItemTooLargeError): + await query(connection, reader, limit=2) + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("size", (200, 1000, 5000)) +def test_large_sparse_kind_directory_reads_no_bodies_or_unbounded_application_candidates(size, monkeypatch): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, _, heads): + async with profile.database.transaction() as connection: + await connection.execute( + insert(ARTIFACTS_TABLE), + [ + { + "scope_id": "scope-a", + "family": "atomic-memory", + "artifact_id": f"m-{index:05d}", + "revision": 1, + "content": b"not a decodable body: directory must not read it", + } + for index in range(size) + ], + ) + await connection.execute( + insert(heads), + [ + { + "scope_id": "scope-a", + "family": "atomic-memory", + "artifact_id": f"m-{index:05d}", + "revision": 1, + "content_kind": "rare" if index % 10 == 0 else "common", + } + for index in range(size) + ], + ) + statements = [] + + def observed(_connection, _cursor, statement, _parameters, _context, _executemany): + statements.append(statement) + + event.listen(profile.database.engine.sync_engine, "before_cursor_execute", observed) + try: + async with profile.database.transaction(consistent_snapshot=True) as connection: + execute = type(connection).execute + returned_rows = [] + + async def measured(current, statement, *args, **kwargs): + result = await execute(current, statement, *args, **kwargs) + if current is connection and result.returns_rows: + frozen = result.freeze() + returned_rows.append(len(frozen.data)) + return frozen() + return result + + monkeypatch.setattr(type(connection), "execute", measured) + page = await query(connection, reader, limit=10, kind="rare") + assert [item.artifact.artifact_id for item in page.items] == [ + f"m-{index:05d}" for index in range(0, 100, 10) + ] + assert page.next_cursor is not None + # Measure actual result cardinality, not SQL text or private call order. + assert returned_rows and max(returned_rows) <= 11 + finally: + event.remove(profile.database.engine.sync_engine, "before_cursor_execute", observed) + # The storage budget prohibits body access regardless of body size/validity. + assert all("pc_artifacts" not in statement for statement in statements) + + asyncio.run(scenario()) + + +def test_missing_metadata_and_generation_overflow_fail_without_publishing_a_page(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, scopes, heads), profile.database.transaction() as connection: + await publish(connection, reader) + await connection.execute(update(heads).values(content_kind=None)) + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await query(connection, reader) + for generation in (-1, (1 << 63) - 1): + await connection.execute(update(scopes).values(atomic_memory_directory_generation=generation)) + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await reader.record_mutation(connection, "scope-a") + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "values", + ( + {"limit": True}, + {"limit": 0}, + {"states": ()}, + {"states": ("active", "active")}, + {"kind": " "}, + {"kind": "x" * 129}, + {"cursor": "x" * 4097}, + ), +) +def test_query_validation_rejects_invalid_contract_inputs(values): + with pytest.raises(ValidationError): + AtomicMemoryDirectoryQuery(scope_id="scope-a", **values) diff --git a/tests/e2e/test_atomic_memory_directory.py b/tests/e2e/test_atomic_memory_directory.py new file mode 100644 index 0000000000..3952cded0d --- /dev/null +++ b/tests/e2e/test_atomic_memory_directory.py @@ -0,0 +1,508 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Directory application, caller transactions and HTTP/SDK/MCP contracts.""" + +from __future__ import annotations + +import asyncio +from contextlib import nullcontext +from dataclasses import replace +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import insert, select + +from powercontext.artifacts.search import ArtifactSearchExecutionContext +from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory +from powercontext.builtin.persistence.artifact_governance import ArtifactGovernanceRepository, ArtifactLifecycleState +from powercontext.builtin.persistence.artifacts import ArtifactRepository +from powercontext.builtin.persistence.atomic_memory_directory import ( + AtomicMemoryDirectoryAuthorizationUnavailableError, + AtomicMemoryDirectoryRepository, + AtomicMemoryDirectoryUnavailableError, +) +from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.sqlite.atomic_memory_index import SQLiteAtomicMemoryIndex +from powercontext.builtin.records import InvalidCursorError +from powercontext.builtin.runtime.atomic_memory import AtomicMemoryApplication +from tests.builtin.persistence.test_atomic_memory_directory import directory_profile, publish, query + + +def test_file_sqlite_bound_transaction_sees_publication_and_rolls_back_directory_state(tmp_path): + async def scenario(): + from powercontext.server.authz import AccessDeniedError + + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile(tmp_path / "directory.sqlite") as (profile, scopes, _): + artifacts = ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader) + scoped = AtomicMemoryApplication(profile.database, artifacts, SQLiteAtomicMemoryIndex(None)).for_scope( + "scope-a" + ) + local = ArtifactSearchExecutionContext(trusted_local=True) + for context in (None, ArtifactSearchExecutionContext()): + with pytest.raises(AccessDeniedError): + await scoped.query_directory(context=context) + uncomposed = AtomicMemoryApplication( + profile.database, ArtifactRepository((AtomicMemory,)), SQLiteAtomicMemoryIndex(None) + ).for_scope("scope-a") + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await uncomposed.query_directory(context=local) + async with profile.database.engine.connect() as connection: + transaction = await connection.begin() + await publish(connection, reader) + page = await scoped.query_directory(context=local, connection=connection) + assert page.items[0].artifact.artifact_id == "a" + assert connection.in_transaction() and not connection.closed + await transaction.rollback() + assert (await scoped.query_directory(context=local)).items == () + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert (await query(connection, reader)).items == () + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == 0 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("provider_name", ("builtin", "casbin")) +@pytest.mark.parametrize("audit_location", ("business", "independent")) +def test_shared_directory_access_preserves_policy_audit_and_global_owner_failure( + provider_name, audit_location, tmp_path, monkeypatch +): + async def scenario(): + from powercontext.server.authz import ( + AccessAuditContext, + AccessBinding, + AccessBindingState, + AccessControlService, + AccessDeniedError, + AccessProviderCapabilities, + AccessRole, + AccessUnavailableError, + BuiltinAuthorizationProvider, + CasbinAuthorizationProvider, + GroupRef, + PrincipalRef, + ResourceRef, + ) + from powercontext.server.authz.models import ArtifactOwnerRelation + from powercontext.server.authz.repository import ( + ACCESS_BINDINGS_TABLE, + ACCESS_TABLES, + RelationalAccessRepository, + _binding_row, + ) + + now = [datetime.now(UTC)] + principal = PrincipalRef(type="user", id="reader") + audit_context = AccessAuditContext(transport="test", operation="directory-query") + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile(tmp_path / "policy.sqlite", extra_tables=ACCESS_TABLES) as (profile, _, _): + relationships = RelationalAccessRepository(profile.database) + provider_type = BuiltinAuthorizationProvider if provider_name == "builtin" else CasbinAuthorizationProvider + scope_binding = await relationships.create_binding( + AccessBinding( + binding_id="scope-reader", + subject=principal, + resource=ResourceRef.scope("scope-a"), + role=AccessRole.SCOPE_VIEWER, + granted_by=principal, + reason="fixture", + created_at=datetime.now(UTC), + expires_at=now[0] + timedelta(minutes=1), + state=AccessBindingState.ACTIVE, + version=1, + policy_revision="pending", + idempotency_key="fixture", + ) + ) + async with profile.database.transaction() as connection: + # Unrelated direct grants in this same Scope must not be loaded + # to decide Scope read. Seed via the shared encoding authority. + await connection.execute( + insert(ACCESS_BINDINGS_TABLE), + [ + _binding_row( + replace( + scope_binding, + binding_id=f"unrelated-{index}", + idempotency_key=f"unrelated-{index}", + role=AccessRole.ARTIFACT_VIEWER, + resource=ResourceRef.artifact( + "scope-a", family="atomic-memory", artifact_id=f"unrelated-{index}" + ), + ) + ) + for index in range(5000) + ], + ) + scoped = AtomicMemoryApplication( + profile.database, + ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader), + SQLiteAtomicMemoryIndex(None), + ).for_scope("scope-a") + # Preserve both default same-file audit and an independent audit store. + async with ( + SQLiteProfile.open(SQLiteConfig(), tables=ACCESS_TABLES) + if audit_location == "independent" + else nullcontext(profile) as audit_profile + ): + audit = RelationalAccessRepository(audit_profile.database) + provider = provider_type(relationships, clock=lambda: now[0]) + access = AccessControlService(provider, relationships=relationships, audit=audit) + context = ArtifactSearchExecutionContext(principal=principal, access=access, audit=audit_context) + async with profile.database.transaction() as connection: + for identity in ("a", "b", "z"): + await publish(connection, reader, identity) + governance = ArtifactGovernanceRepository(atomic_memory_directory=reader) + current = await governance.get(connection, "scope-a", "atomic-memory", "z") + await governance.transition_merge( + connection, "scope-a", "atomic-memory", "z", current, ArtifactLifecycleState.RETIRED + ) + for identity in ("a", "b"): + await relationships.establish_artifact_owner( + ArtifactOwnerRelation( + resource=ResourceRef.artifact("scope-a", family="atomic-memory", artifact_id=identity), + owner=principal, + established_at=datetime.now(UTC), + policy_revision="0", + idempotency_key=identity, + ) + ) + # Even the active first page fails for an unowned retired row elsewhere. + with pytest.raises(AccessUnavailableError, match="owner"): + await scoped.query_directory(limit=1, context=context) + with pytest.raises(AccessDeniedError): + await scoped.query_directory( + context=replace(context, principal=PrincipalRef(type="user", id="stranger")) + ) + await relationships.establish_artifact_owner( + ArtifactOwnerRelation( + resource=ResourceRef.artifact("scope-a", family="atomic-memory", artifact_id="z"), + owner=principal, + established_at=datetime.now(UTC), + policy_revision="0", + idempotency_key="z", + ) + ) + # Measure actual application row materialization, including + # authorization, instead of inferring cost from selector LIMIT. + async with profile.database.engine.connect() as observed: + execute = type(observed).execute + returned_rows = [] + + async def measured(current, statement, *args, **kwargs): + result = await execute(current, statement, *args, **kwargs) + if result.returns_rows: + frozen = result.freeze() + returned_rows.append(len(frozen.data)) + return frozen() + return result + + with monkeypatch.context() as patch: + patch.setattr(type(observed), "execute", measured) + first = await scoped.query_directory(limit=1, context=context) + assert returned_rows and max(returned_rows) <= 2 + assert [item.artifact.artifact_id for item in first.items] == ["a"] and first.next_cursor + async with ( + access.defer_decision_audit(), + profile.database.transaction(consistent_snapshot=True) as connection, + ): + tail = await scoped.query_directory( + limit=1, cursor=first.next_cursor, context=context, connection=connection + ) + assert connection.in_transaction() and not connection.closed + assert [item.artifact.artifact_id for item in tail.items] == ["b"] and tail.next_cursor is None + # Host-resolved groups and configured provider are cursor context, + # even when a direct Scope grant still permits both pages. + with pytest.raises(InvalidCursorError): + await scoped.query_directory( + limit=1, + cursor=first.next_cursor, + context=replace( + context, audit=replace(audit_context, subject_groups=(GroupRef("group", "g"),)) + ), + ) + alternate_type = ( + CasbinAuthorizationProvider if provider_name == "builtin" else BuiltinAuthorizationProvider + ) + alternate = AccessControlService( + alternate_type(relationships), relationships=relationships, audit=audit + ) + with pytest.raises(InvalidCursorError): + await scoped.query_directory( + limit=1, cursor=first.next_cursor, context=replace(context, access=alternate) + ) + disabled = AccessControlService( + provider, + relationships=relationships, + audit=audit, + provider_capabilities=AccessProviderCapabilities(False, True, True), + ) + with pytest.raises(AtomicMemoryDirectoryAuthorizationUnavailableError): + await scoped.query_directory(context=replace(context, access=disabled)) + assert access.audit is audit and access.provider is provider and provider._repository is relationships + assert await audit.list_audit(resource=ResourceRef.scope("scope-a"), limit=10) + now[0] += timedelta(minutes=2) + with pytest.raises(AccessDeniedError): + await scoped.query_directory(limit=1, cursor=first.next_cursor, context=context) + now[0] -= timedelta(minutes=2) + await relationships.create_binding( + AccessBinding( + binding_id="another-reader", + subject=PrincipalRef(type="user", id="other"), + resource=ResourceRef.scope("scope-a"), + role=AccessRole.SCOPE_VIEWER, + granted_by=principal, + reason="fixture policy change", + created_at=now[0], + expires_at=None, + state=AccessBindingState.ACTIVE, + version=1, + policy_revision="pending", + idempotency_key="another-reader", + ) + ) + with pytest.raises(InvalidCursorError): + await scoped.query_directory(limit=1, cursor=first.next_cursor, context=context) + first = await scoped.query_directory(limit=1, context=context) + await relationships.revoke_binding( + "scope-reader", + revoked_by=principal, + expected_version=1, + idempotency_key="revoke", + revoked_at=datetime.now(UTC), + ) + # Revocation is checked before trusting a still-signed continuation. + with pytest.raises(AccessDeniedError): + await scoped.query_directory(limit=1, cursor=first.next_cursor, context=context) + + asyncio.run(scenario()) + + +def test_http_sdk_and_mcp_share_compact_pages_authentication_and_restart_semantics(monkeypatch): + async def scenario(): + from types import SimpleNamespace + + import httpx + from fastmcp import Client + from fastmcp.client.transports import StreamableHttpTransport + from starlette.middleware import Middleware + + from powercontext.builtin.persistence import atomic_memory_directory as directory + from powercontext.builtin.persistence.tags import RelationalTagService + from powercontext.builtin.tags import ArtifactTagTarget + from powercontext.client import PowerContextClient + from powercontext.client.errors import ServerResponseError + from powercontext.http import QueryAtomicMemoryDirectoryRequest + from powercontext.server.app import create_app + from powercontext.server.authentication import StaticBearerAuthenticationProvider + from powercontext.server.authz import ( + AccessBinding, + AccessBindingState, + AccessControlService, + AccessRole, + BuiltinAuthorizationProvider, + PrincipalRef, + ResourceRef, + ) + from powercontext.server.authz.models import ArtifactOwnerRelation + from powercontext.server.authz.repository import ACCESS_TABLES, RelationalAccessRepository + from powercontext.server.mcp import mount_mcp + from powercontext.server.middleware import AuthenticationMiddleware + + principal = PrincipalRef(type="user", id="reader") + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile(extra_tables=ACCESS_TABLES) as (profile, _, _): + artifacts = ArtifactRepository((AtomicMemory,), atomic_memory_directory=reader) + atomic = AtomicMemoryApplication(profile.database, artifacts, SQLiteAtomicMemoryIndex(None)) + relationships = RelationalAccessRepository(profile.database) + await relationships.create_binding( + AccessBinding( + binding_id="viewer", + subject=principal, + resource=ResourceRef.scope("scope-a"), + role=AccessRole.SCOPE_VIEWER, + granted_by=principal, + reason="fixture", + created_at=datetime.now(UTC), + expires_at=None, + state=AccessBindingState.ACTIVE, + version=1, + policy_revision="pending", + idempotency_key="viewer", + ) + ) + tags = tuple(chr(0x4E00 + index) * 64 for index in range(16)) + tag_service = RelationalTagService(profile.database, artifacts=artifacts) + async with profile.database.transaction() as connection: + for identity in ("a", "b"): + await publish(connection, reader, identity, text="Private body must not enter the directory.") + for identity in ("a", "b"): + await relationships.establish_artifact_owner( + ArtifactOwnerRelation( + resource=ResourceRef.artifact("scope-a", family="atomic-memory", artifact_id=identity), + owner=principal, + established_at=datetime.now(UTC), + policy_revision="0", + idempotency_key=identity, + ) + ) + target = ArtifactTagTarget(family="atomic-memory", artifact_id=identity) + current = await tag_service.get("scope-a", target) + await tag_service.replace("scope-a", target, tags, expected_etag=current.etag) + access = AccessControlService( + BuiltinAuthorizationProvider(relationships), relationships=relationships, audit=relationships + ) + authentication = StaticBearerAuthenticationProvider("fixture-token", principal) + app = create_app( + application=SimpleNamespace(atomic_memory=atomic), + access_control=access, + authentication_provider=authentication, + middleware=(Middleware(AuthenticationMiddleware, provider=authentication),), + ) + mount_mcp(app) + url = "http://127.0.0.1" + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=url) as http: + credential = "fixture-token" + sdk = PowerContextClient(url, token=credential, http_client=http) + request = QueryAtomicMemoryDirectoryRequest.model_validate({ + "scope_id": "scope-a", + "limit": 1, + "tags": list(tags), + }) + first = await sdk.query_atomic_memory_directory(request) + assert first.next_cursor and len(first.next_cursor) <= 4096 + first_json = first.model_dump(mode="json") + assert set(first_json) == {"scope_id", "items", "next_cursor"} + assert set(first_json["items"][0]) == {"artifact", "kind", "state", "state_version", "merged_into_id"} + assert first_json["items"][0]["artifact"] == { + "family": "atomic-memory", + "artifact_id": "a", + "revision": 1, + } + assert "Private body" not in first.model_dump_json() + + def create_http_client(headers=None, timeout=None, auth=None, **_): + return httpx.AsyncClient( + transport=transport, + base_url=url, + headers={"Authorization": "Bearer fixture-token", **(headers or {})}, + timeout=timeout, + auth=auth, + follow_redirects=True, + ) + + mcp_transport = StreamableHttpTransport(f"{url}/mcp/", httpx_client_factory=create_http_client) + async with app.router.lifespan_context(app), Client(mcp_transport) as mcp: + tool = next(tool for tool in await mcp.list_tools() if tool.name == "query_atomic_memory_directory") + assert tool.annotations and tool.annotations.readOnlyHint + continuation = request.model_copy(update={"cursor": first.next_cursor}).model_dump(mode="json") + result = await mcp.call_tool("query_atomic_memory_directory", continuation) + assert not result.is_error + assert result.data["items"][0]["artifact"]["artifact_id"] == "b" + assert result.data["next_cursor"] is None + # An actual tag write invalidates both transports' old cursor. + target = ArtifactTagTarget(family="atomic-memory", artifact_id="a") + current = await tag_service.get("scope-a", target) + await tag_service.replace("scope-a", target, (), expected_etag=current.etag) + invalid = await mcp.call_tool("query_atomic_memory_directory", continuation, raise_on_error=False) + assert invalid.is_error and "invalid_cursor" in invalid.content[0].text + with pytest.raises(ServerResponseError) as invalid: + await sdk.query_atomic_memory_directory(request.model_copy(update={"cursor": first.next_cursor})) + assert invalid.value.status_code == 400 and invalid.value.code == "invalid_cursor" + # Single-item failure is mapped without returning a partial page. + with monkeypatch.context() as patch: + patch.setattr(directory, "DIRECTORY_ITEM_BYTES", 1) + with pytest.raises(ServerResponseError) as oversized: + await sdk.query_atomic_memory_directory(QueryAtomicMemoryDirectoryRequest(scope_id="scope-a")) + assert oversized.value.status_code == 413 + assert oversized.value.code == "atomic_memory_directory_item_too_large" + # Wire constraints are enforced as well as domain validation. + for changes in ({"limit": True}, {"states": []}, {"states": ["active", "active"]}, {"kind": " "}): + response = await http.post( + "/v1/atomic-memory/directory/query", + json={"scope_id": "scope-a", **changes}, + headers={"Authorization": "Bearer fixture-token"}, + ) + assert response.status_code == 422 and "items" not in response.json() + unauthenticated = await http.post("/v1/atomic-memory/directory/query", json={"scope_id": "scope-a"}) + assert unauthenticated.status_code == 401 + from powercontext.server.authz import AccessProviderCapabilities + + app.state.access_control = AccessControlService( + access.provider, + relationships=relationships, + audit=relationships, + provider_capabilities=AccessProviderCapabilities(False, True, True), + ) + try: + with pytest.raises(ServerResponseError) as unsupported: + await sdk.query_atomic_memory_directory(QueryAtomicMemoryDirectoryRequest(scope_id="scope-a")) + assert unsupported.value.status_code == 503 + assert unsupported.value.code == "atomic_memory_directory_authorization_unavailable" + finally: + app.state.access_control = access + await relationships.revoke_binding( + "viewer", + revoked_by=principal, + expected_version=1, + idempotency_key="revoke", + revoked_at=datetime.now(UTC), + ) + with pytest.raises(ServerResponseError) as revoked: + await sdk.query_atomic_memory_directory(QueryAtomicMemoryDirectoryRequest(scope_id="scope-a")) + assert revoked.value.status_code == 403 + audit = await relationships.list_audit(resource=ResourceRef.scope("scope-a"), limit=100) + decisions = [event for event in audit if event.operation == "query_atomic_memory_directory"] + assert {event.transport for event in decisions} >= {"http", "mcp"} + assert all(event.principal == principal for event in decisions) + + asyncio.run(scenario()) + + +def test_uncomposed_http_directory_fails_closed_and_full_record_tagged_list_still_works(): + async def scenario(): + from types import SimpleNamespace + + import httpx + + from powercontext.builtin.persistence.tags import RelationalTagService + from powercontext.builtin.tags import ArtifactTagTarget + from powercontext.server.app import create_app + + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, _, _): + async with profile.database.transaction() as connection: + await publish(connection, reader, text="Full record text remains available.") + artifacts = ArtifactRepository((AtomicMemory,)) + tag_service = RelationalTagService(profile.database, artifacts=artifacts) + target = ArtifactTagTarget(family="atomic-memory", artifact_id="a") + current = await tag_service.get("scope-a", target) + await tag_service.replace("scope-a", target, ("PRODUCTION",), expected_etag=current.etag) + atomic = AtomicMemoryApplication(profile.database, artifacts, SQLiteAtomicMemoryIndex(None)) + app = create_app(application=SimpleNamespace(atomic_memory=atomic), access_mode="disabled") + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://127.0.0.1") as http: + response = await http.post("/v1/atomic-memory/directory/query", json={"scope_id": "scope-a"}) + assert response.status_code == 503 + assert response.json()["error"]["code"] == "atomic_memory_directory_unavailable" + response = await http.post( + "/v1/atomic-memory/list", json={"scope_id": "scope-a", "tags": ["production"]} + ) + assert response.status_code == 200 + assert response.json()["items"][0]["text"] == "Full record text remains available." + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_directory_migration.py b/tests/e2e/test_atomic_memory_directory_migration.py new file mode 100644 index 0000000000..40db34e7dc --- /dev/null +++ b/tests/e2e/test_atomic_memory_directory_migration.py @@ -0,0 +1,383 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Real business baselines and explicit unified directory migration/recovery.""" + +from __future__ import annotations + +import asyncio +import json +import sqlite3 +from pathlib import Path + +import pytest +from pydantic import SecretStr +from sqlalchemy import Engine, event +from typer.testing import CliRunner + +from powercontext.artifacts.search import ArtifactSearchExecutionContext +from powercontext.builtin.persistence.atomic_memory_directory import AtomicMemoryDirectoryUnavailableError +from powercontext.builtin.persistence.migrations import MigrationError +from powercontext.builtin.persistence.migrations.deployment import deployment_runner, production_bundle +from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner +from powercontext.builtin.persistence.sqlite import SQLiteConfig +from powercontext.builtin.records import InvalidCursorError +from powercontext.builtin.runtime.composition import open_builtin_contexts +from powercontext.builtin.runtime.config import BuiltinConfig +from powercontext.cli.app import create_cli +from powercontext.server.cli import app as server_cli +from powercontext.server.factory import create_server_app +from powercontext.server.settings import ServerSettings + +_LOCAL = ArtifactSearchExecutionContext(trusted_local=True) + + +def _settings(path: Path) -> ServerSettings: + return ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{path}"), + cursor_signing_secret=SecretStr("directory-migration-test-secret-32-bytes"), + ) + + +def _runner(path: Path) -> SQLiteMigrationRunner: + runner = deployment_runner(_settings(path), target="atomic-memory-directory") + assert isinstance(runner, SQLiteMigrationRunner) + return runner + + +def _apply(runner: SQLiteMigrationRunner): + plan = runner.plan() + return runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True) + + +async def _source(path: Path, count: int = 2) -> str: + application = create_server_app(settings=_settings(path), scheduler_path=path.with_suffix(".scheduler.json")) + async with application.router.lifespan_context(application): + runtime = application.state.application + scope_id = (await runtime.scopes.default_scope()).scope_id + await runtime._records().create_atomic_memories( + scope_id, tuple({"kind": "fact", "text": f"Fact {index}."} for index in range(count)) + ) + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await runtime.atomic_memory.for_scope(scope_id).query_directory(context=_LOCAL) + return scope_id + + +def test_shared_cli_initializes_directory_target_and_inspections_do_not_create_it(tmp_path): + path = tmp_path / "fresh.sqlite" + environment = tmp_path / "deployment.env" + environment.write_text( + f"POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\nPOWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{path}\n" + ) + command = ["server", "db-migrate"] + options = ["--target", "atomic-memory-directory", "--env-file", str(environment)] + cli = CliRunner() + for action in ("status", "plan"): + result = cli.invoke(create_cli([server_cli]), [*command, action, *options]) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data_tasks"] == ["atomic-memory-directory-v1"] + assert not path.exists() + assert not path.with_name(path.name + ".pc-migration-state").exists() + plan = json.loads(result.output) + applied = cli.invoke( + create_cli([server_cli]), + [ + *command, + "apply", + *options, + "--backup", + "auto", + "--plan-id", + plan["plan_id"], + "--yes", + "--maintenance-confirmed", + ], + ) + assert applied.exit_code == 0, applied.output + assert json.loads(applied.output)["revision"] == "p0005" + verified = cli.invoke(create_cli([server_cli]), [*command, "verify", *options]) + assert verified.exit_code == 0, verified.output + assert json.loads(verified.output)["server_ready"] is False + + +def test_actual_server_baseline_upgrades_without_changing_exact_history_or_identity(tmp_path): + path = tmp_path / "existing.sqlite" + scope_id = asyncio.run(_source(path, 201)) + with sqlite3.connect(path) as connection: + originals = connection.execute("SELECT * FROM pc_artifacts ORDER BY artifact_id,revision").fetchall() + identity = connection.execute("SELECT server_id FROM pc_server_identity").fetchone() + runner = _runner(path) + plan = runner.plan() + assert plan.adopt_baseline and plan.source_revision == "p0004" + assert plan.data_tasks == ("atomic-memory-directory-v1",) + result = _apply(runner) + assert result.backup_state == "completed" + assert runner.verify().state == "ready" + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT * FROM pc_artifacts ORDER BY artifact_id,revision").fetchall() == originals + assert connection.execute("SELECT server_id FROM pc_server_identity").fetchone() == identity + assert connection.execute("SELECT DISTINCT content_kind FROM pc_artifact_heads").fetchall() == [("fact",)] + generation = connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] + assert generation == 2 + assert _apply(runner).changed is False + with sqlite3.connect(path) as connection: + assert ( + connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] == generation + ) + + async def browse(): + application = create_server_app(settings=_settings(path), scheduler_path=path.with_suffix(".scheduler.json")) + async with application.router.lifespan_context(application): + runtime = application.state.application + scoped = runtime.atomic_memory.for_scope(scope_id) + first = await scoped.query_directory(limit=100, context=_LOCAL) + second = await scoped.query_directory(limit=100, cursor=first.next_cursor, context=_LOCAL) + last = await scoped.query_directory(limit=100, cursor=second.next_cursor, context=_LOCAL) + assert len(first.items + second.items + last.items) == 201 + assert last.next_cursor is None + item = first.items[0] + exact = await scoped.get(item.artifact.artifact_id, revision=item.artifact.revision, context=_LOCAL) + assert exact.ref == item.artifact + await runtime._records().create_atomic_memories(scope_id, ({"kind": "rule", "text": "A new rule."},)) + with pytest.raises(InvalidCursorError): + await scoped.query_directory(limit=100, cursor=first.next_cursor, context=_LOCAL) + + asyncio.run(browse()) + assert runner.verify().state == "ready" + + +def test_committed_first_batch_resumes_with_original_backup_and_no_extra_generation(tmp_path, monkeypatch): + path = tmp_path / "interrupted.sqlite" + asyncio.run(_source(path, 201)) + runner = _runner(path) + load = runner.bundle.task_functions + + def interrupted(task): + functions = load(task) + execute = functions["apply_batches"] + + def crash(connection): + for changed in execute(connection): + yield changed + raise RuntimeError("interrupted after committed data batch") # noqa: TRY003 + + functions["apply_batches"] = crash + return functions + + monkeypatch.setattr(runner.bundle, "task_functions", interrupted) + with pytest.raises(RuntimeError, match="committed data batch"): + _apply(runner) + evidence = json.loads(runner.evidence_path.read_text()) + assert evidence["state"] == "active" + + async def cannot_restart_during_maintenance(): + from powercontext.server.authz.composition import open_builtin_access_control + from powercontext.server.authz.models import PrincipalRef + + application = create_server_app(settings=_settings(path), scheduler_path=path.with_suffix(".scheduler.json")) + with pytest.raises(MigrationError, match="recovery_required"): + async with application.router.lifespan_context(application): + pytest.fail("Unfinished offline maintenance must not start business writers") + with pytest.raises(MigrationError, match="recovery_required"): + async with open_builtin_access_control( + _settings(path).database, bootstrap_administrators=(PrincipalRef(type="user", id="new-admin"),) + ): + pytest.fail("Authorization bootstrap must not write before the maintenance gate") + + asyncio.run(cannot_restart_during_maintenance()) + with sqlite3.connect(path) as connection: + assert ( + connection.execute("SELECT COUNT(*) FROM pc_artifact_heads WHERE content_kind IS NULL").fetchone()[0] == 1 + ) + assert connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] == 1 + resumed = _runner(path) + assert resumed.plan().state == "recovery_required" + assert resumed.plan().revisions == () + assert resumed.plan().data_tasks == ("atomic-memory-directory-v1",) + _apply(resumed) + completed = json.loads(resumed.evidence_path.read_text()) + assert completed["backup"] == evidence["backup"] + assert completed["maintenance_window_id"] == evidence["maintenance_window_id"] + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] == 2 + assert _apply(resumed).changed is False + + +def test_bad_later_batch_preserves_that_batch_and_explicit_repair_can_resume(tmp_path): + path = tmp_path / "bad-body.sqlite" + asyncio.run(_source(path, 201)) + with sqlite3.connect(path) as connection: + identity, body = connection.execute( + "SELECT artifact_id,content FROM pc_artifacts ORDER BY artifact_id DESC LIMIT 1" + ).fetchone() + connection.execute("UPDATE pc_artifacts SET content=? WHERE artifact_id=?", (b"not-json", identity)) + runner = _runner(path) + with pytest.raises(MigrationError, match="verification_failed"): + _apply(runner) + + async def cannot_restart_after_failed_task(): + application = create_server_app(settings=_settings(path), scheduler_path=path.with_suffix(".scheduler.json")) + with pytest.raises(MigrationError, match="recovery_required"): + async with application.router.lifespan_context(application): + pytest.fail("Failed offline maintenance must remain stopped") + + asyncio.run(cannot_restart_after_failed_task()) + with sqlite3.connect(path) as connection: + assert connection.execute( + "SELECT content_kind FROM pc_artifact_heads WHERE artifact_id=?", (identity,) + ).fetchone() == (None,) + assert ( + connection.execute("SELECT COUNT(*) FROM pc_artifact_heads WHERE content_kind='fact'").fetchone()[0] == 200 + ) + connection.execute("UPDATE pc_artifacts SET content=? WHERE artifact_id=?", (body, identity)) + _apply(_runner(path)) + assert _runner(path).verify().state == "ready" + + +def test_completed_schema_does_not_hide_data_damage_and_disabled_queries_keep_writer_maintenance(tmp_path): + path = tmp_path / "damaged-kind.sqlite" + scope_id = asyncio.run(_source(path)) + runner = _runner(path) + _apply(runner) + with sqlite3.connect(path) as connection: + connection.execute( + "UPDATE pc_artifact_heads SET content_kind='wrong' WHERE artifact_id=(SELECT MIN(artifact_id) FROM pc_artifact_heads)" + ) + plan = runner.plan() + assert plan.revisions == () and plan.state == "migration_required" + with pytest.raises(MigrationError, match="migration_required"): + runner.verify() + + async def write_while_directory_unavailable(): + application = create_server_app(settings=_settings(path), scheduler_path=path.with_suffix(".scheduler.json")) + async with application.router.lifespan_context(application): + runtime = application.state.application + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await runtime.atomic_memory.for_scope(scope_id).query_directory(context=_LOCAL) + await runtime._records().create_atomic_memories( + scope_id, ({"kind": "rule", "text": "Still maintain new writes."},) + ) + + asyncio.run(write_while_directory_unavailable()) + with sqlite3.connect(path) as connection: + assert connection.execute("SELECT COUNT(*) FROM pc_artifact_heads WHERE content_kind='rule'").fetchone()[0] == 1 + before = connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] + _apply(runner) + with sqlite3.connect(path) as connection: + assert ( + connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] == before + 1 + ) + + +def test_unknown_business_objects_are_not_ignored_and_missing_recovery_evidence_blocks(tmp_path, monkeypatch): + path = tmp_path / "unknown.sqlite" + asyncio.run(_source(path)) + with sqlite3.connect(path) as connection: + connection.execute("CREATE TABLE unknown_extension (value TEXT)") + with pytest.raises(MigrationError, match="unknown_baseline"): + _runner(path).plan() + with sqlite3.connect(path) as connection: + connection.execute("DROP TABLE unknown_extension") + runner = _runner(path) + original = runner.bundle.task_functions + + def fail(task): + functions = original(task) + functions["apply_batches"] = lambda connection: (_ for _ in ()).throw(RuntimeError("interrupted")) + return functions + + monkeypatch.setattr(runner.bundle, "task_functions", fail) + with pytest.raises(RuntimeError, match="interrupted"): + _apply(runner) + runner.evidence_path.unlink() + with pytest.raises(MigrationError, match="recovery_required"): + _runner(path).plan() + + +def test_default_acceptance_bundle_remains_four_table_scope(): + assert production_bundle().head == "p0003" + assert production_bundle().data_tasks == () + + +def test_unsupported_sqlite_connection_options_cannot_silently_skip_managed_writer_hooks(tmp_path): + path = tmp_path / "connection-options.sqlite" + scope_id = asyncio.run(_source(path)) + _apply(_runner(path)) + with sqlite3.connect(path) as connection: + before = connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] + + async def mutate_through_alias(): + alias = SQLiteConfig(url=f"sqlite+aiosqlite:///{path}?timeout=5") + async with open_builtin_contexts(BuiltinConfig(database=alias)) as contexts: + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + await contexts.atomic_memory.for_scope(scope_id).query_directory(context=_LOCAL) + await contexts.records.create_atomic_memories(scope_id, ({"kind": "rule", "text": "Maintain aliases."},)) + + asyncio.run(mutate_through_alias()) + with sqlite3.connect(path) as connection: + assert ( + connection.execute("SELECT atomic_memory_directory_generation FROM pc_scopes").fetchone()[0] == before + 1 + ) + assert _runner(path).verify().state == "ready" + + +def test_directory_target_rejects_existing_acceptance_database_before_any_write(tmp_path): + path = tmp_path / "acceptance.sqlite" + acceptance = SQLiteMigrationRunner(path, production_bundle()) + _apply(acceptance) + before = path.read_bytes() + with pytest.raises(MigrationError, match="unsupported_target"): + _runner(path).plan() + assert path.read_bytes() == before + + +def test_registered_sqlite_task_target_rejects_other_backends_before_connecting(tmp_path): + from powercontext.builtin.persistence.seekdb import SeekDBConfig + + settings = ServerSettings(database=SeekDBConfig(path=tmp_path / "not-created")) + with pytest.raises(MigrationError, match="unsupported_target"): + deployment_runner(settings, target="atomic-memory-directory") + assert not (tmp_path / "not-created").exists() + + +def test_directory_target_requires_the_evidence_location_its_runtime_can_inspect(tmp_path): + path = tmp_path / "not-created.sqlite" + with pytest.raises(MigrationError, match="unsupported_option"): + deployment_runner(_settings(path), target="atomic-memory-directory", evidence_dir=tmp_path / "other-evidence") + assert not path.exists() + assert not (tmp_path / "other-evidence").exists() + + +def test_topic_child_does_not_scan_atomic_bodies_when_reusing_managed_parent_schema(tmp_path): + path = tmp_path / "topic-child.sqlite" + asyncio.run(_source(path)) + _apply(_runner(path)) + statements = [] + + def observe(_connection, _cursor, statement, _parameters, _context, _many): + normalized = statement.lower() + if "left join pc_artifacts" in normalized and "atomic-memory" in normalized: + statements.append(statement) + + async def open_child(): + async with open_builtin_contexts(BuiltinConfig(database=_settings(path).database), _topic_memory_worker=True): + pass + + event.listen(Engine, "before_cursor_execute", observe) + try: + asyncio.run(open_child()) + finally: + event.remove(Engine, "before_cursor_execute", observe) + assert statements == [], "Topic-only children must not verify the unrelated Atomic directory" diff --git a/tests/e2e/test_mcp_transport.py b/tests/e2e/test_mcp_transport.py index 31aaa22b79..6db485c46d 100644 --- a/tests/e2e/test_mcp_transport.py +++ b/tests/e2e/test_mcp_transport.py @@ -253,6 +253,7 @@ async def exercise_tools() -> set[str]: "list_artifact_candidates", "list_atomic_memories", "list_memory_entries", + "query_atomic_memory_directory", "list_scopes", "merge_atomic_memories", "change_atomic_memory_lifecycle", diff --git a/tests/test_mcp.py b/tests/test_mcp.py index d21641ba0f..00a9025bba 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -109,6 +109,7 @@ async def inspect_components() -> tuple[list[str], int, int]: "get_artifact_revision", "replace_artifact", "get_atomic_memory_state", + "query_atomic_memory_directory", "list_atomic_memories", "search_atomic_memory", "merge_atomic_memories", @@ -158,6 +159,7 @@ async def inspect_annotations() -> dict[str, Any]: "get_artifact", "get_artifact_revision", "get_atomic_memory_state", + "query_atomic_memory_directory", "list_atomic_memories", "search_atomic_memory", "preview_atomic_memory_restoration", From f5b06e8013e2480bf1b7dbf9b23d9adb72596b17 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:28:38 +0800 Subject: [PATCH 12/14] fix(integrations): synchronize Atomic directory contracts and fixtures --- .../src-tauri/src/transport/operations.json | 2 +- desktop/ui/src/generated/api.d.ts | 106 ++++++++++++++++++ desktop/ui/src/generated/operations.ts | 2 +- evaluation/skills/skill-up/README.md | 2 +- .../skill-up/evals/cases/empty-search.yaml | 2 + .../skill-up/evals/cases/explicit-save.yaml | 1 + .../skill-up/evals/cases/failed-save.yaml | 1 + .../evals/cases/inspect-candidates.yaml | 1 + .../skill-up/evals/cases/ordinary-coding.yaml | 1 + .../fixtures/mcp/powercontext-default.json | 3 + .../mcp/powercontext-failed-save.json | 3 + .../powercontext-tool-contract/CLAUDE.md | 5 + .../dsh/plugins/powercontext/lib/index.js | 11 ++ 13 files changed, 137 insertions(+), 3 deletions(-) diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 82cee3a518..99465c19b8 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "7aa91ae9af78300e7bdbcc2e8b2135da5749469290a1aee67bc255c3e9a07b0c", + "contractSha256": "ddd2b39df34f1de02d0d529376fd49362dc8cf4bfcc874f3db6ae095f7729b5b", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index 1ed9b6b95a..273132d6f3 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -16,6 +16,26 @@ // Generated from openapi/powercontext.yaml. Do not edit. export interface paths { + "/v1/atomic-memory/directory/query": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Query compact current Atomic Memory references + * @description Returns current metadata and exact revisions without loading Memory text or history. Any directory or tag change invalidates continuation; restart from the first page on invalid_cursor or cursor_expired. Requires verified directory migration and writer composition, and a supported local authorization configuration. Production enablement is unavailable until the unified migration bundle manages the complete target. Unavailable codes include artifact_owner_pending, atomic_memory_directory_authorization_unavailable and atomic_memory_directory_unavailable; do not treat failure as an empty page. + */ + post: operations["query_atomic_memory_directory"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/atomic-memory/list": { parameters: { query?: never; @@ -4779,6 +4799,35 @@ export interface components { items: components["schemas"]["AccessAuditEvent"][]; next_cursor: string | null; }; + QueryAtomicMemoryDirectoryRequest: { + scope_id: string; + /** + * @default [ + * "active" + * ] + */ + states: components["schemas"]["AtomicMemoryState"][]; + kind?: string | null; + /** @default [] */ + tags: string[]; + /** @default all */ + tag_match: components["schemas"]["TagMatch"]; + /** @default 50 */ + limit: number; + cursor?: string | null; + }; + AtomicMemoryDirectoryItem: { + artifact: components["schemas"]["ArtifactReference"]; + kind: string; + state: components["schemas"]["AtomicMemoryState"]; + state_version: number; + merged_into_id: string | null; + }; + QueryAtomicMemoryDirectoryResponse: { + scope_id: string; + items: components["schemas"]["AtomicMemoryDirectoryItem"][]; + next_cursor: string | null; + }; }; responses: { /** @description The request query or pagination cursor is invalid. */ @@ -4919,6 +4968,63 @@ export interface components { } export type $defs = Record; export interface operations { + query_atomic_memory_directory: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["QueryAtomicMemoryDirectoryRequest"]; + }; + }; + responses: { + /** @description Compact current directory page. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["QueryAtomicMemoryDirectoryResponse"]; + }; + }; + /** @description The cursor is invalid or its query, identity, policy or directory generation changed. */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + /** @description The cursor expired; restart from the first page. */ + 410: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + /** @description A compact item exceeds the response budget; no partial page is returned. */ + 413: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorResponse"]; + }; + }; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; list_atomic_memories: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index ea299aa267..821c5f634d 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "7aa91ae9af78300e7bdbcc2e8b2135da5749469290a1aee67bc255c3e9a07b0c"; +export const contractSha256 = "ddd2b39df34f1de02d0d529376fd49362dc8cf4bfcc874f3db6ae095f7729b5b"; export const operations = { "get_liveness": { "method": "GET", diff --git a/evaluation/skills/skill-up/README.md b/evaluation/skills/skill-up/README.md index b87a114b49..e8b4014a26 100644 --- a/evaluation/skills/skill-up/README.md +++ b/evaluation/skills/skill-up/README.md @@ -101,7 +101,7 @@ All relative config paths start at this project root, not at `evals/`. Both arms receive the same `evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md`: required/optional top-level argument names, types and defaults derived from OpenAPI and its MCP specialization, including content CAS headers. -The mock catalog contains the Server's complete declared MCP tool subset (54 operations), verified against +The mock catalog contains the Server's complete declared MCP tool subset (55 operations), verified against `src/powercontext/server/mcp.py` and the generated operation IDs. The successful save reply contains Atomic `records`; empty search returns the current `text` mode and an empty `hits` list. This fills the generic mock schemas' parameter information gap diff --git a/evaluation/skills/skill-up/evals/cases/empty-search.yaml b/evaluation/skills/skill-up/evals/cases/empty-search.yaml index d7c6753bd4..71bd8ff442 100644 --- a/evaluation/skills/skill-up/evals/cases/empty-search.yaml +++ b/evaluation/skills/skill-up/evals/cases/empty-search.yaml @@ -69,6 +69,7 @@ judge: - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_memory_entries} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_scopes} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__publish_artifact} + - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_atomic_memory_directory} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_code} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__record_task_outcome} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__reject_artifact_candidate} @@ -99,6 +100,7 @@ judge: - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__list_memory_entries} - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__list_scopes} - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__publish_artifact} + - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__query_atomic_memory_directory} - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__query_code} - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__record_task_outcome} - tool_not_called_in_turn: {turn: 2, name: mcp__powercontext__reject_artifact_candidate} diff --git a/evaluation/skills/skill-up/evals/cases/explicit-save.yaml b/evaluation/skills/skill-up/evals/cases/explicit-save.yaml index 8e9cf90aab..ee37a43916 100644 --- a/evaluation/skills/skill-up/evals/cases/explicit-save.yaml +++ b/evaluation/skills/skill-up/evals/cases/explicit-save.yaml @@ -62,6 +62,7 @@ judge: - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_memory_entries} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_scopes} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__publish_artifact} + - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_atomic_memory_directory} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_code} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__record_task_outcome} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__reject_artifact_candidate} diff --git a/evaluation/skills/skill-up/evals/cases/failed-save.yaml b/evaluation/skills/skill-up/evals/cases/failed-save.yaml index c6a121d9db..3b50ae7917 100644 --- a/evaluation/skills/skill-up/evals/cases/failed-save.yaml +++ b/evaluation/skills/skill-up/evals/cases/failed-save.yaml @@ -78,6 +78,7 @@ judge: - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_memory_entries} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_scopes} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__publish_artifact} + - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_atomic_memory_directory} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_code} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__record_task_outcome} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__reject_artifact_candidate} diff --git a/evaluation/skills/skill-up/evals/cases/inspect-candidates.yaml b/evaluation/skills/skill-up/evals/cases/inspect-candidates.yaml index 5b1e0d2fca..d2a0b45e5f 100644 --- a/evaluation/skills/skill-up/evals/cases/inspect-candidates.yaml +++ b/evaluation/skills/skill-up/evals/cases/inspect-candidates.yaml @@ -64,6 +64,7 @@ judge: - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_memory_entries} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_scopes} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__publish_artifact} + - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_atomic_memory_directory} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_code} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__record_task_outcome} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__reject_artifact_candidate} diff --git a/evaluation/skills/skill-up/evals/cases/ordinary-coding.yaml b/evaluation/skills/skill-up/evals/cases/ordinary-coding.yaml index 26b8ca4f1a..7f516a4bb2 100644 --- a/evaluation/skills/skill-up/evals/cases/ordinary-coding.yaml +++ b/evaluation/skills/skill-up/evals/cases/ordinary-coding.yaml @@ -54,6 +54,7 @@ judge: - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_memory_entries} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__list_scopes} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__publish_artifact} + - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_atomic_memory_directory} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__query_code} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__record_task_outcome} - tool_not_called_in_turn: {turn: 1, name: mcp__powercontext__reject_artifact_candidate} diff --git a/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-default.json b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-default.json index a2695ac5b3..1f2abbf867 100644 --- a/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-default.json +++ b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-default.json @@ -172,6 +172,9 @@ "publish_artifact": { "error": "unexpected" }, + "query_atomic_memory_directory": { + "error": "unexpected" + }, "query_code": { "error": "unexpected" }, diff --git a/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-failed-save.json b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-failed-save.json index b5cbbc182b..261336f447 100644 --- a/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-failed-save.json +++ b/evaluation/skills/skill-up/evals/fixtures/mcp/powercontext-failed-save.json @@ -172,6 +172,9 @@ "publish_artifact": { "error": "unexpected" }, + "query_atomic_memory_directory": { + "error": "unexpected" + }, "query_code": { "error": "unexpected" }, diff --git a/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md index 2f518c1ca6..f4d5ed6305 100644 --- a/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md +++ b/evaluation/skills/skill-up/evals/fixtures/repos/powercontext-tool-contract/CLAUDE.md @@ -210,6 +210,11 @@ not replace those schemas or enforce their constraints. It contains no scenario - Required: `idempotency_key: string`, `source: ArtifactAddress`, `target_scope_id: string`. - Optional: (none). +## mcp__powercontext__query_atomic_memory_directory + +- Required: `scope_id: string`. +- Optional: `cursor: string | null`, `kind: string | null`, `limit: integer = 50`, `states: array<"active" | "forgotten" | "merged" | "retired"> = ["active"]`, `tag_match: "all" | "any" = "all"`, `tags: array = []`. + ## mcp__powercontext__query_code - Required: `operation: CodeStatusOperation | CodeChangesOperation | CodeMapOperation | CodeSearchOperation | CodeRelationOperation | CodeTestsOperation | CodeReadOperation`, `scope_id: string`. diff --git a/integrations/dsh/plugins/powercontext/lib/index.js b/integrations/dsh/plugins/powercontext/lib/index.js index 5c527a7a21..dbeddd2f2f 100644 --- a/integrations/dsh/plugins/powercontext/lib/index.js +++ b/integrations/dsh/plugins/powercontext/lib/index.js @@ -159,6 +159,17 @@ function writeFailureConfirmation(error) { //#endregion //#region src/operations.generated.ts const OPERATIONS$1 = { + query_atomic_memory_directory: { + method: "POST", + path: "/v1/atomic-memory/directory/query", + location: "body", + scopeMode: "current", + pathParameters: [], + queryParams: [], + headerParams: [], + successStatuses: [200], + emptyStatuses: [] + }, list_atomic_memories: { method: "POST", path: "/v1/atomic-memory/list", From a479b19c2f0798ac1254cd768ceeb5d46fb4fa96 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:30:42 +0800 Subject: [PATCH 13/14] refactor(memory): remove superseded directory repair entry point --- .../persistence/atomic_memory_directory.py | 95 +------------------ .../test_atomic_memory_directory.py | 70 -------------- 2 files changed, 1 insertion(+), 164 deletions(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_directory.py b/src/powercontext/builtin/persistence/atomic_memory_directory.py index 86020c6297..8a8eb73ef7 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_directory.py +++ b/src/powercontext/builtin/persistence/atomic_memory_directory.py @@ -34,9 +34,8 @@ from powercontext.artifacts import ArtifactRef from powercontext.builtin.artifacts.atomic_memory.models import AtomicMemory, AtomicMemoryContent, AtomicMemoryState from powercontext.builtin.persistence.atomic_memory import atomic_memory_state_expression -from powercontext.builtin.persistence.codec import load_model from powercontext.builtin.persistence.cursor_codec import SignedCursorCodec -from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE, identity_string +from powercontext.builtin.persistence.tables import ARTIFACT_HEADS_TABLE, identity_string from powercontext.builtin.persistence.tags import tag_predicate from powercontext.builtin.records import BaseAccessError from powercontext.builtin.sources import validate_scope_id @@ -263,98 +262,6 @@ async def record_publication(self, connection: AsyncConnection, scope_id: str, a raise AtomicMemoryDirectoryUnavailableError await self.record_mutation(connection, scope_id) - async def maintain_batch( # noqa: C901 - self, - connection: AsyncConnection, - scope_id: str, - *, - after: str = "", - batch_size: int = 200, - repair: bool = False, - ) -> tuple[str | None, int]: - """Validate or repair current metadata for an offline domain-task executor. - - Return (next position, changed heads). The caller owns maintenance locks, - backup, schema readiness, transactions and the full-Scope iteration. - Verification never writes. Repair reads exact current content only and - advances generation only when this batch actually changes metadata. - """ - - validate_scope_id(scope_id) - if type(batch_size) is not int or not 1 <= batch_size <= 200 or type(repair) is not bool: - raise ValueError("invalid directory maintenance batch") # noqa: TRY003 - if after: - ArtifactRef(family="atomic-memory", artifact_id=after, revision=1) - if repair: - await self.lock_scope(connection, scope_id) - generation = await connection.scalar( - select(_SCOPES.c.atomic_memory_directory_generation).where(_SCOPES.c.scope_id == scope_id) - ) - if type(generation) is not int or not 0 <= generation <= _MAX_GENERATION: - raise AtomicMemoryDirectoryUnavailableError - heads, artifacts = ARTIFACT_HEADS_TABLE, ARTIFACTS_TABLE - rows = ( - ( - await connection.execute( - select( - heads.c.artifact_id, - heads.c.revision, - _KIND.label("kind"), - atomic_memory_state_expression().label("state"), - heads.c.governance_generation.label("state_version"), - heads.c.merged_into_id, - heads.c.replacement_artifact_id, - artifacts.c.content, - ) - .select_from( - heads.outerjoin( - artifacts, - (artifacts.c.scope_id == heads.c.scope_id) - & (artifacts.c.family == heads.c.family) - & (artifacts.c.artifact_id == heads.c.artifact_id) - & (artifacts.c.revision == heads.c.revision), - ) - ) - .where( - heads.c.scope_id == scope_id, - heads.c.family == "atomic-memory", - heads.c.artifact_id > after, - ) - .order_by(heads.c.artifact_id) - .limit(batch_size + 1) - ) - ) - .mappings() - .all() - ) - changes = [] - # Validate the entire batch before changing any head. An outer join - # ensures a missing exact body cannot disappear from verification. - for row in rows[:batch_size]: - content = load_model(AtomicMemoryContent, row["content"], kind="artifact", name="atomic-memory") - _directory_item({**row, "kind": content.kind}) - if row["kind"] != content.kind: - if not repair: - raise AtomicMemoryDirectoryUnavailableError - changes.append((row["artifact_id"], row["revision"], content.kind)) - for identity, revision, kind in changes: - result = await connection.execute( - update(_HEADS) - .where( - _HEADS.c.scope_id == scope_id, - _HEADS.c.family == "atomic-memory", - _HEADS.c.artifact_id == identity, - _HEADS.c.revision == revision, - ) - .values(content_kind=kind) - ) - if result.rowcount != 1: - raise AtomicMemoryDirectoryUnavailableError - if changes: - await self.record_mutation(connection, scope_id) - next_after = str(rows[batch_size - 1]["artifact_id"]) if len(rows) > batch_size else None - return next_after, len(changes) - async def record_mutation(self, connection: AsyncConnection, scope_id: str) -> None: """Invalidate cursors after an effective state/tag/repair mutation; never commit.""" diff --git a/tests/builtin/persistence/test_atomic_memory_directory.py b/tests/builtin/persistence/test_atomic_memory_directory.py index 5d38825e70..3483a99218 100644 --- a/tests/builtin/persistence/test_atomic_memory_directory.py +++ b/tests/builtin/persistence/test_atomic_memory_directory.py @@ -524,76 +524,6 @@ async def scenario(): asyncio.run(scenario()) -def test_directory_maintenance_batches_are_readonly_idempotent_and_preserve_exact_history(tmp_path): - async def scenario(): - reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) - async with directory_profile(tmp_path / "maintenance.sqlite") as (profile, scopes, heads): - async with profile.database.transaction() as connection: - originals = [await publish(connection, reader, identity) for identity in ("a", "b", "c")] - page = await query(connection, reader, limit=1) - generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) - await connection.execute(update(heads).values(content_kind=None)) - # Plan/verify is read-only even when it detects incomplete metadata. - async with profile.database.transaction(consistent_snapshot=True) as connection: - with pytest.raises(AtomicMemoryDirectoryUnavailableError): - await reader.maintain_batch(connection, "scope-a", batch_size=2) - assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation - assert (await connection.execute(select(heads.c.content_kind))).scalars().all() == [None] * 3 - # Interrupt after one committed batch and restart from the beginning. - async with profile.database.transaction() as connection: - assert await reader.maintain_batch(connection, "scope-a", batch_size=2, repair=True) == ("b", 2) - async with profile.database.transaction() as connection: - assert await reader.maintain_batch(connection, "scope-a", batch_size=2, repair=True) == ("b", 0) - assert await reader.maintain_batch(connection, "scope-a", after="b", batch_size=2, repair=True) == ( - None, - 1, - ) - assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation + 2 - async with profile.database.transaction(consistent_snapshot=True) as connection: - assert await reader.maintain_batch(connection, "scope-a", batch_size=2) == ("b", 0) - assert await reader.maintain_batch(connection, "scope-a", after="b", batch_size=2) == (None, 0) - with pytest.raises(InvalidCursorError): - await query(connection, reader, limit=1, cursor=page.next_cursor) - for original in originals: - assert ( - await ArtifactRepository((AtomicMemory,)).get(connection, "scope-a", original.as_ref()) - == original - ) - - asyncio.run(scenario()) - - -def test_directory_maintenance_validates_whole_batch_and_caller_can_roll_back_repair(tmp_path): - async def scenario(): - from powercontext.builtin.persistence.errors import InvalidStoredPayloadError - - reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) - async with directory_profile(tmp_path / "repair.sqlite") as (profile, scopes, heads): - async with profile.database.transaction() as connection: - for identity in ("a", "b"): - await publish(connection, reader, identity) - await connection.execute(update(heads).values(content_kind="incorrect")) - await connection.execute( - update(ARTIFACTS_TABLE).where(ARTIFACTS_TABLE.c.artifact_id == "b").values(content=b"broken") - ) - generation = await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) - async with profile.database.transaction() as connection: - with pytest.raises(InvalidStoredPayloadError): - await reader.maintain_batch(connection, "scope-a", repair=True) - assert (await connection.execute(select(heads.c.content_kind))).scalars().all() == ["incorrect"] * 2 - assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation - async with profile.database.engine.connect() as connection: - transaction = await connection.begin() - assert await reader.maintain_batch(connection, "scope-a", batch_size=1, repair=True) == ("a", 1) - assert connection.in_transaction() and not connection.closed - await transaction.rollback() - async with profile.database.transaction(consistent_snapshot=True) as connection: - assert (await connection.execute(select(heads.c.content_kind))).scalars().all() == ["incorrect"] * 2 - assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == generation - - asyncio.run(scenario()) - - @pytest.mark.parametrize("size", (200, 1000, 5000)) def test_scope_owner_ready_uses_one_bounded_result_and_exact_canonical_owner_keys(size, record_property, monkeypatch): async def scenario(): From 571253eb33391ffdac82256e2c7f0573840de616 Mon Sep 17 00:00:00 2001 From: wutongyuonce <147830929+wutongyuonce@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:12:10 +0800 Subject: [PATCH 14/14] fix(memory): reject noninteger SQLite directory generations --- .../persistence/atomic_memory_directory.py | 4 +++- .../test_atomic_memory_directory.py | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/powercontext/builtin/persistence/atomic_memory_directory.py b/src/powercontext/builtin/persistence/atomic_memory_directory.py index 8a8eb73ef7..4f18219fbb 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_directory.py +++ b/src/powercontext/builtin/persistence/atomic_memory_directory.py @@ -27,7 +27,7 @@ import rfc8785 from pydantic import BaseModel, ConfigDict, Field, StrictInt, ValidationError, field_validator -from sqlalchemy import BigInteger, ColumnElement, column, literal_column, select, table, update +from sqlalchemy import BigInteger, ColumnElement, column, func, literal_column, select, table, update from sqlalchemy.engine import RowMapping from sqlalchemy.ext.asyncio import AsyncConnection @@ -147,6 +147,7 @@ async def lock_scope(self, connection: AsyncConnection, scope_id: str) -> None: update(_SCOPES) .where( _SCOPES.c.scope_id == scope_id, + func.typeof(_SCOPES.c.atomic_memory_directory_generation) == "integer", _SCOPES.c.atomic_memory_directory_generation >= 0, _SCOPES.c.atomic_memory_directory_generation <= _MAX_GENERATION, ) @@ -269,6 +270,7 @@ async def record_mutation(self, connection: AsyncConnection, scope_id: str) -> N update(_SCOPES) .where( _SCOPES.c.scope_id == scope_id, + func.typeof(_SCOPES.c.atomic_memory_directory_generation) == "integer", _SCOPES.c.atomic_memory_directory_generation >= 0, _SCOPES.c.atomic_memory_directory_generation < _MAX_GENERATION, ) diff --git a/tests/builtin/persistence/test_atomic_memory_directory.py b/tests/builtin/persistence/test_atomic_memory_directory.py index 3483a99218..f43092f3e0 100644 --- a/tests/builtin/persistence/test_atomic_memory_directory.py +++ b/tests/builtin/persistence/test_atomic_memory_directory.py @@ -478,6 +478,27 @@ async def scenario(): asyncio.run(scenario()) +def test_fractional_sqlite_generation_cannot_commit_business_writes_or_be_incremented(): + async def scenario(): + reader = AtomicMemoryDirectoryRepository(SignedCursorCodec(secret=b"fixture")) + async with directory_profile() as (profile, scopes, heads): + async with profile.database.transaction() as connection: + await publish(connection, reader) + # SQLite BIGINT affinity and a nonnegative CHECK still admit fractions. + await connection.execute(update(scopes).values(atomic_memory_directory_generation=0.5)) + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + async with profile.database.transaction() as connection: + await publish(connection, reader, "must-not-commit") + with pytest.raises(AtomicMemoryDirectoryUnavailableError): + async with profile.database.transaction() as connection: + await reader.record_mutation(connection, "scope-a") + async with profile.database.transaction(consistent_snapshot=True) as connection: + assert (await connection.execute(select(heads.c.artifact_id))).scalars().all() == ["a"] + assert await connection.scalar(select(scopes.c.atomic_memory_directory_generation)) == 0.5 + + asyncio.run(scenario()) + + def test_other_scopes_and_families_do_not_invalidate_atomic_directory(): async def scenario(): from tests.builtin.persistence.contract import Handoff, HandoffContent, HandoffDraft