diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 260c25ea7c..868559ab98 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -124,6 +124,7 @@ jobs: tests/test_file_lock_cross_process.py tests/control_plane/test_coordination_file_provider.py tests/control_plane/test_effect_runtime_integration.py + tests/control_plane/test_local_authority_shadow_outbox.py tests/test_self_update_runtime_activation.py tests/test_windows_install.py diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md index 5f81324c9a..4983f38222 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md @@ -2134,14 +2134,14 @@ write that cannot preserve the contract. fence, run `promote`, render the projections, and record the declaration of question 11; refuse a rerun whose source digest changed unless the abandoned store is explicitly discarded. -- A transaction-bound capture (question 14): the runtime shadow samples the - source after the commit, from outside the writer's lock, so a concurrent - writer can land inside the sampled projection and a crash between the commit - and the dispatch loses the mirror. The parity-half outbox closes both: the - prepared entry is written inside the lock the writer already holds, the - committed marker after the primary write returns, and a bounded drain turns - each entry into exactly one shadow transaction whose `operation_id` is the - entry id. `qualify` then counts entries, not samples. +- Full transaction-capture qualification (question 14): Todo add, update, + complete, supersede, and archive plus native lease acquire, renew, transfer, + release, auto-acquire, and fence-close now emit prepared/committed outbox + entries around their primary write. The bounded drain commits complete + versioned records into the existing `coordination.runtime_shadow` file-v0 + lineage; it does not create the former second local-shadow candidate. Before + promotion, add sustained mixed-writer parity runs, event-only Todo coverage, + and the selected provider profile's recovery/capacity evidence. - The provider-neutral authority binding, compatibility projection outbox, and conformance rows for file, NoKV, and PostgreSQL. This does not require all providers to promote together; each profile must pass the same contract @@ -2172,6 +2172,6 @@ one document is acceptable only for promotion bootstrap and bounded test goals. | Lane | May start | Scope and exit condition | Dependency | | --- | --- | --- | --- | | P. PostgreSQL provider plane | Now, from current `main` | Keep the existing `AuthorityStore` contract; finish schema migration/install ownership, authenticated service and tenant authorization, restore-incarnation rotation, pool/cancellation/failover behavior, and reviewed indexes, partitioning, retention, and measured capacity. Live PostgreSQL conformance remains mandatory. | Does not depend on #3870 and must not stack on its branch. This lane alone creates no runtime caller or promotion claim. | -| C. Canonical transaction capture | Now, by revising or replacing #3870 | Make the outbox transport complete versioned Todo/lease records into `coordination.runtime_shadow.commit`; retire the duplicate observation/local-shadow lineage and prove omission/explicit-clear behavior. | Can run in parallel with P, but both C and the selected provider profile must finish before parity or promotion integration. | +| C. Canonical transaction capture | In implementation, based on #3870 | Transaction-bound outbox capture now targets the one `coordination.runtime_shadow` lineage and retains complete versioned Todo/lease records. Finish sustained mixed-writer parity, explicit-clear/omission coverage, and event-only Todo recovery evidence. | Can run in parallel with P, but both C and the selected provider profile must finish before parity or promotion integration. | | I. Binding and qualification integration | After P and C | Bind one exact provider lineage, field manifest, source revision, digest, and cursor; run sustained transaction parity and provider-specific recovery/capacity qualification without consulting legacy state for missing fields. | This is the merge point between provider work and capture work. | | F. Promotion and cleanup | After I and explicit maintainer approval | Add provider-first CLI routing, the lock-owning promotion orchestrator, compatibility projection outbox, post-promotion fenced export/rollback, then delete duplicate reference aggregates and flip the reviewed stage/hold declarations. | No profile is eligible until its exact implementation and lineage pass P, C, and I. | diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md index 8340d170dc..7c97b00268 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md @@ -1698,11 +1698,12 @@ integrity chain、确定性 scan 与 recovery readback。物理 profile 可以 片）：取 Todo 与 lease 两把 legacy 锁，要求 `qualify` 在当前 revision 与 digest 上 为 `qualified`，engage fence，执行 `promote`，渲染投影，写入问题 11 的声明；源 digest 已变时拒绝重跑，除非显式丢弃被放弃的 store。 -- 事务绑定的捕获（问题 14）：runtime shadow 在提交之后、写者锁之外采样源，并发写者 - 可能落进被采样的投影，commit 与 dispatch 之间崩溃则丢失镜像。parity 半段的 outbox - 同时关闭两者：prepared entry 在写者已持有的锁内写入，committed 标记在主写返回后 - 写入，有界 drain 把每条 entry 变成恰好一笔 `operation_id` 为 entry id 的 shadow - 事务。此后 `qualify` 数的是 entry，不是采样。 +- 完成事务捕获资格验证（问题 14）：Todo add、update、complete、supersede、archive， + 以及 native lease acquire、renew、transfer、release、auto-acquire、fence-close，现已在 + 主写前后生成 prepared/committed outbox entry。有界 drain 把完整、带版本的 record + 提交到既有 `coordination.runtime_shadow` file-v0 lineage，不再创建第二套 local-shadow + candidate。晋升前仍需补持续 mixed-writer parity、event-only Todo 覆盖和所选 provider + profile 的 recovery/capacity 证据。 - provider-neutral authority binding、兼容投影 outbox，以及 file、NoKV、PostgreSQL 的 conformance row。三个 provider 不必同时晋升，但每个 profile 都必须先通过同一 合同才具备资格。 @@ -1727,6 +1728,6 @@ latency、response size 与 recovery。单文档全量保留只适用于 promoti | Lane | 何时开始 | 范围与退出条件 | 依赖 | | --- | --- | --- | --- | | P. PostgreSQL provider plane | 现在，从当前 `main` 开始 | 保持既有 `AuthorityStore` 合同；完成 schema migration/install ownership、authenticated service 与 tenant authorization、restore-incarnation rotation、pool/cancellation/failover 行为，以及经评审的 index、partition、retention 与实测 capacity。真实 PostgreSQL conformance 始终是强制门禁。 | 不依赖 #3870，也不得叠在其分支上。仅完成本 lane 不产生 runtime caller 或 promotion 声明。 | -| C. Canonical transaction capture | 现在，通过修订或替代 #3870 | 让 outbox 把完整、带版本的 Todo/lease record 传给 `coordination.runtime_shadow.commit`；退役重复 observation/local-shadow lineage，并证明 omission/explicit-clear 行为。 | 可与 P 并行；但 C 与选定 provider profile 都完成后，才能进入 parity 或 promotion 集成。 | +| C. Canonical transaction capture | 实现中，基于 #3870 | transaction-bound outbox 已指向唯一 `coordination.runtime_shadow` lineage，并保留完整带版本的 Todo/lease record；继续完成 sustained mixed-writer parity、explicit-clear/omission 与 event-only Todo recovery 证据。 | 可与 P 并行；但 C 与选定 provider profile 都完成后，才能进入 parity 或 promotion 集成。 | | I. Binding 与资格集成 | P 与 C 完成后 | 绑定一个精确 provider lineage、field manifest、source revision、digest 与 cursor；运行持续 transaction parity，以及 provider-specific recovery/capacity qualification；缺字段时不得查询 legacy state 补齐。 | 这是 provider 工作与 capture 工作的汇合点。 | | F. Promotion 与清理 | I 完成且 maintainer 显式批准后 | 加入 provider-first CLI routing、持锁 promotion orchestrator、兼容投影 outbox、晋升后 fenced export/rollback；随后删除重复 reference aggregate，并翻转经评审的 stage/hold 声明。 | 一个 profile 的精确实现与 lineage 通过 P、C、I 前，不具备晋升资格。 | diff --git a/loopx/cli.py b/loopx/cli.py index 441f6821a5..8dea8f8510 100644 --- a/loopx/cli.py +++ b/loopx/cli.py @@ -100,6 +100,7 @@ handle_support_control_command, handle_handoff_mode_command, handle_task_lease_command, + handle_authority_shadow_command, handle_version_command, handle_host_mode_plan_command, handle_worker_bridge_command, @@ -140,6 +141,7 @@ register_support_control_commands, register_handoff_mode_command, register_task_lease_command, + register_authority_shadow_command, register_todo_command, register_version_command, register_host_mode_plan_command, @@ -325,6 +327,7 @@ def build_parser() -> LoopXArgumentParser: register_todo_command(sub, add_subcommand_format) register_coordination_shadow_command(sub, add_subcommand_format) register_task_lease_command(sub, add_subcommand_format) + register_authority_shadow_command(sub, add_subcommand_format) register_handoff_mode_command(sub, add_subcommand_format) register_shared_goal_alignment_command(sub, add_subcommand_format) register_quota_command(sub) @@ -814,6 +817,16 @@ def main(argv: list[str] | None = None) -> int: if task_lease_result is not None: return task_lease_result + authority_shadow_result = handle_authority_shadow_command( + args, + registry_path=registry_path, + runtime_root_arg=args.runtime_root, + output_format=output_format, + print_payload=print_payload, + ) + if authority_shadow_result is not None: + return authority_shadow_result + handoff_mode_result = handle_handoff_mode_command( args, registry_path=registry_path, diff --git a/loopx/cli_commands/__init__.py b/loopx/cli_commands/__init__.py index c9be2c6d58..ac76f6242a 100644 --- a/loopx/cli_commands/__init__.py +++ b/loopx/cli_commands/__init__.py @@ -142,6 +142,10 @@ def _load_exports() -> None: handle_support_control_command, register_support_control_commands, ) + from .authority_shadow import ( + handle_authority_shadow_command, + register_authority_shadow_command, + ) from .task_lease import handle_task_lease_command, register_task_lease_command from .todo import handle_todo_command, register_todo_command from .version import handle_version_command, register_version_command @@ -221,6 +225,7 @@ def _load_exports() -> None: "handle_starter_visible_pilot_command", "handle_summary_all_command", "handle_support_control_command", + "handle_authority_shadow_command", "handle_task_lease_command", "handle_todo_command", "handle_version_command", @@ -268,6 +273,7 @@ def _load_exports() -> None: "register_summary_all_command", "register_status_commands", "register_support_control_commands", + "register_authority_shadow_command", "register_task_lease_command", "register_todo_command", "register_version_command", diff --git a/loopx/cli_commands/authority_shadow.py b/loopx/cli_commands/authority_shadow.py new file mode 100644 index 0000000000..e62a3b4f67 --- /dev/null +++ b/loopx/cli_commands/authority_shadow.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import argparse +from collections.abc import Callable +from pathlib import Path + +from ..control_plane.coordination.local_authority_shadow_adapter import ( + CLI_DRAIN_LOCK_TIMEOUT_SECONDS, + drain_local_authority_shadow_outbox, + effective_runtime_root, + local_authority_shadow_status, +) +from ..control_plane.coordination.local_authority_shadow_outbox import OutboxError +from ..file_lock import LockAcquireTimeoutError + + +AUTHORITY_SHADOW_CLI_SCHEMA = "loopx_authority_shadow_cli_v0" +CLI_DRAIN_MAX_ENTRIES = 256 +CLI_DRAIN_BUDGET_SECONDS = 30.0 + +PrintPayload = Callable[ + [dict[str, object], str, Callable[[dict[str, object]], str]], + None, +] + + +_DRAIN_FIELDS = ( + "outcome", + "reason_code", + "delivered", + "replayed", + "reconciled", + "no_op", + "reseeded", + "reclaimed_residue", + "pending_after", + "prepared_only_after", + "budget_exhausted", + "last_cursor", + "candidate_readback_verified", +) + + +def _drain_markdown_lines(payload: dict[str, object]) -> list[str]: + lines = [f"- {key}: `{payload.get(key)}`" for key in _DRAIN_FIELDS] + stopped_at = payload.get("stopped_at") + if isinstance(stopped_at, dict): + lines.append( + "- stopped_at: " + f"`{stopped_at.get('partition')}#{stopped_at.get('seq')}` " + f"→ `{stopped_at.get('outcome')}` ({stopped_at.get('reason_code')})" + ) + return lines + + +def _status_markdown_lines(payload: dict[str, object]) -> list[str]: + lines: list[str] = [] + config = payload.get("config") + if isinstance(config, dict): + lines.append(f"- config: `{config.get('status')}`") + backlog = payload.get("outbox") + partitions = backlog.items() if isinstance(backlog, dict) else [] + for partition, facts in partitions: + if isinstance(facts, dict): + lines.append( + f"- outbox.{partition}: committed_pending=" + f"`{facts.get('committed_pending')}` prepared_only=" + f"`{facts.get('prepared_only')}` cursor_last_seq=" + f"`{facts.get('cursor_last_seq')}`" + ) + candidate = payload.get("candidate") + if isinstance(candidate, dict): + lines.append( + f"- candidate: status=`{candidate.get('status')}` cursor=" + f"`{candidate.get('cursor')}` store_identity=`{candidate.get('store_identity')}`" + ) + lines.append(f"- store_bytes: `{payload.get('store_bytes')}`") + lines.append(f"- retention_pressure: `{payload.get('retention_pressure')}`") + return lines + + +def render_authority_shadow_markdown(payload: dict[str, object]) -> str: + lines = [ + "# LoopX Authority Shadow", + "", + f"- ok: `{payload.get('ok')}`", + f"- action: `{payload.get('action')}`", + f"- goal_id: `{payload.get('goal_id')}`", + ] + if payload.get("error"): + lines.append(f"- error: {payload.get('error')}") + if payload.get("error_code"): + lines.append(f"- error_code: `{payload.get('error_code')}`") + if payload.get("action") == "drain": + lines.extend(_drain_markdown_lines(payload)) + elif payload.get("action") == "status": + lines.extend(_status_markdown_lines(payload)) + return "\n".join(lines) + "\n" + + +def register_authority_shadow_command( + subparsers: argparse._SubParsersAction, + add_subcommand_format: Callable[[argparse.ArgumentParser], None], +) -> None: + parser = subparsers.add_parser( + "authority-shadow", + help=( + "Drain or inspect the transaction-bound local authority shadow outbox " + "for one goal. The candidate store is evidence only; it never decides." + ), + ) + add_subcommand_format(parser) + parser.add_argument( + "authority_shadow_command", + choices=["drain", "status"], + help="drain delivers pending outbox entries; status reports backlog and candidate facts.", + ) + parser.add_argument("--goal-id", required=True, help="Goal id whose shadow outbox to operate on.") + parser.add_argument( + "--max-entries", + type=int, + default=CLI_DRAIN_MAX_ENTRIES, + help=f"Maximum entries one drain pass delivers (default {CLI_DRAIN_MAX_ENTRIES}).", + ) + parser.add_argument( + "--budget-seconds", + type=float, + default=CLI_DRAIN_BUDGET_SECONDS, + help=f"Wall-clock budget for one drain pass (default {CLI_DRAIN_BUDGET_SECONDS:g}).", + ) + parser.add_argument( + "--lock-timeout-seconds", + type=float, + default=CLI_DRAIN_LOCK_TIMEOUT_SECONDS, + help=( + "How long to wait for the per-goal drain lock before reporting drain_deferred " + f"(default {CLI_DRAIN_LOCK_TIMEOUT_SECONDS:g})." + ), + ) + + +def handle_authority_shadow_command( + args: argparse.Namespace, + *, + registry_path: Path, + runtime_root_arg: str | None, + output_format: Callable[..., str], + print_payload: PrintPayload, +) -> int | None: + if args.command != "authority-shadow": + return None + action = str(getattr(args, "authority_shadow_command", None)) + payload: dict[str, object] + try: + # The same resolver every writer hook uses, so drain and status address + # the lineage those hooks wrote. + runtime_root = effective_runtime_root(registry_path, runtime_root_arg) + if action == "drain": + if args.max_entries < 1: + raise ValueError("--max-entries must be at least 1") + result = drain_local_authority_shadow_outbox( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=args.goal_id, + max_entries=args.max_entries, + budget_seconds=args.budget_seconds, + lock_timeout_seconds=args.lock_timeout_seconds, + ) + payload = { + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": "drain", + **result.to_payload(), + } + else: + payload = { + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + **local_authority_shadow_status( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=args.goal_id, + ), + } + except OutboxError as exc: + payload = { + "ok": False, + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": action, + "goal_id": args.goal_id, + "error": str(exc), + "error_code": exc.reason_code, + } + except LockAcquireTimeoutError as exc: + payload = { + "ok": False, + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": action, + "goal_id": args.goal_id, + "error": str(exc), + **exc.to_payload(), + } + except Exception as exc: + payload = { + "ok": False, + "schema_version": AUTHORITY_SHADOW_CLI_SCHEMA, + "action": action, + "goal_id": args.goal_id, + "error": str(exc), + "error_code": exc.__class__.__name__, + } + print_payload(payload, output_format(args), render_authority_shadow_markdown) + return 0 if payload.get("ok") else 1 + + +__all__ = [ + "AUTHORITY_SHADOW_CLI_SCHEMA", + "handle_authority_shadow_command", + "register_authority_shadow_command", + "render_authority_shadow_markdown", +] diff --git a/loopx/control_plane/agents/agent_scope.py b/loopx/control_plane/agents/agent_scope.py index 9edd1d7d6a..9802ec2f1a 100644 --- a/loopx/control_plane/agents/agent_scope.py +++ b/loopx/control_plane/agents/agent_scope.py @@ -2,7 +2,7 @@ import re from dataclasses import dataclass -from typing import Any +from typing import Any, TypeGuard from .agent_scope_frontier import ( AGENT_LANE_FRONTIER_HINT_SCHEMA_VERSION, @@ -87,15 +87,12 @@ } AGENT_TASK_SCOPE = "goal_all_read_claimed_run_global_read_v0" - - -def _agent_identity_has_scoped_lane(agent_identity: dict[str, Any] | None) -> bool: - return bool( - isinstance(agent_identity, dict) - and normalize_todo_claimed_by(agent_identity.get("agent_id")) +def _agent_identity_has_scoped_lane( + agent_identity: dict[str, Any] | None, +) -> TypeGuard[dict[str, object]]: + return isinstance(agent_identity, dict) and bool( + normalize_todo_claimed_by(agent_identity.get("agent_id")) ) - - def _attach_agent_identity_contracts( *, payload: dict[str, Any], @@ -110,9 +107,7 @@ def _attach_agent_identity_contracts( def _todo_task_class(item: dict[str, Any]) -> str: return todo_item_task_class(item) - - -def _todo_projection_sort_key(item: dict[str, Any]) -> tuple[int, int, int, str]: +def _todo_projection_sort_key(item: dict[str, Any]) -> tuple[int, int]: return todo_projection_sort_key(item) @@ -377,17 +372,12 @@ def _scoped_user_gate_fallback( gates = _open_user_gate_todo_items(user_todo_summary) if not gates or not isinstance(agent_todo_summary, dict): return None - - due_monitor_candidates = ( - agent_todo_summary.get("monitor_due_items") - if isinstance(agent_todo_summary.get("monitor_due_items"), list) - else [] - ) - ready_deferred_candidates = ( - agent_todo_summary.get("deferred_resume_candidates") - if isinstance(agent_todo_summary.get("deferred_resume_candidates"), list) - else [] + raw_due_monitor_candidates = agent_todo_summary.get("monitor_due_items") + due_monitor_candidates = raw_due_monitor_candidates if isinstance(raw_due_monitor_candidates, list) else [] + raw_ready_deferred_candidates = agent_todo_summary.get( + "deferred_resume_candidates" ) + ready_deferred_candidates = raw_ready_deferred_candidates if isinstance(raw_ready_deferred_candidates, list) else [] # An empty capability projection is authoritative for advancement work. # Due monitors are projected separately from advancement candidates and are @@ -404,11 +394,13 @@ def _scoped_user_gate_fallback( *ready_deferred_candidates, ] else: + raw_backlog_items = agent_todo_summary.get("executable_backlog_items") + raw_first_items = agent_todo_summary.get("first_executable_items") advancement_items = ( - agent_todo_summary.get("executable_backlog_items") - if isinstance(agent_todo_summary.get("executable_backlog_items"), list) - else agent_todo_summary.get("first_executable_items") - if isinstance(agent_todo_summary.get("first_executable_items"), list) + raw_backlog_items + if isinstance(raw_backlog_items, list) + else raw_first_items + if isinstance(raw_first_items, list) else [] ) executable_items = [ @@ -518,11 +510,8 @@ def _scoped_user_gate_fallback( def _first_executable_todo_text(agent_todo_summary: dict[str, Any] | None) -> str | None: if not isinstance(agent_todo_summary, dict): return None - items = ( - agent_todo_summary.get("first_executable_items") - if isinstance(agent_todo_summary.get("first_executable_items"), list) - else [] - ) + raw_items = agent_todo_summary.get("first_executable_items") + items = raw_items if isinstance(raw_items, list) else [] for item in items: if not isinstance(item, dict): continue @@ -540,7 +529,8 @@ def _first_monitor_todo_text(agent_todo_summary: dict[str, Any] | None) -> str | if not isinstance(agent_todo_summary, dict): return None for key in ("monitor_due_items", "monitor_open_items"): - items = agent_todo_summary.get(key) if isinstance(agent_todo_summary.get(key), list) else [] + raw_items = agent_todo_summary.get(key) + items = raw_items if isinstance(raw_items, list) else [] for item in items: if not isinstance(item, dict): continue @@ -946,7 +936,8 @@ def _agent_scope_monitor_blocked_resume_candidates( continue if item.get("resume_ready") is not False: continue - condition = item.get("resume_condition") if isinstance(item.get("resume_condition"), dict) else {} + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} if normalize_todo_status(condition.get("target_status")) != TODO_STATUS_OPEN: continue target_todo_id = normalize_todo_id( diff --git a/loopx/control_plane/agents/capability_gate.py b/loopx/control_plane/agents/capability_gate.py index 5e9786f188..eaac3df43e 100644 --- a/loopx/control_plane/agents/capability_gate.py +++ b/loopx/control_plane/agents/capability_gate.py @@ -315,16 +315,16 @@ def _collect_capability_gate_candidates( agent_todo_summary: dict[str, Any], ) -> tuple[list[dict[str, Any]], str]: raw_items, source = _select_advancement_candidate_source(agent_todo_summary) + raw_due_monitor_items = agent_todo_summary.get("monitor_due_items") due_monitor_items = ( - agent_todo_summary.get("monitor_due_items") - if isinstance(agent_todo_summary.get("monitor_due_items"), list) - else [] + raw_due_monitor_items if isinstance(raw_due_monitor_items, list) else [] + ) + raw_blocked_due_monitor_items = agent_todo_summary.get( + "monitor_capability_blocked_due_items" ) blocked_due_monitor_items = ( - agent_todo_summary.get("monitor_capability_blocked_due_items") - if isinstance( - agent_todo_summary.get("monitor_capability_blocked_due_items"), list - ) + raw_blocked_due_monitor_items + if isinstance(raw_blocked_due_monitor_items, list) else [] ) monitor_sources: list[str] = [] diff --git a/loopx/control_plane/coordination/local_authority_shadow.ts b/loopx/control_plane/coordination/local_authority_shadow.ts index b455cf5e04..1ccb21cddc 100644 --- a/loopx/control_plane/coordination/local_authority_shadow.ts +++ b/loopx/control_plane/coordination/local_authority_shadow.ts @@ -1,13 +1,25 @@ +import { createHash } from "node:crypto"; import { join } from "node:path"; import type { JsonObject } from "../effect_program.ts"; import { EffectRuntimeRequestError } from "../effect_runtime_errors.ts"; -import { requireJsonObject, requireNonEmptyString } from "../runtime_decode.ts"; +import { + requireInteger, + requireJsonObject, + requireNonEmptyString, + requireStringLiteral, +} from "../runtime_decode.ts"; import type { AuthorityStore, + AuthorityStoreCommitResult, + AuthorityStoreCommittedTransaction, AuthorityStoreLoadResult, AuthorityStoreReceiptResult, } from "./authority_store.ts"; +import { authorityUnicodeCompare, canonicalAuthorityBytes } from "./authority_store_codec.ts"; +import { + TODO_CANONICAL_READ_RECORD_FIELDS, +} from "./coordination_projection.ts"; import { FileAuthorityStore } from "./file_authority_store.ts"; export const LOCAL_AUTHORITY_SHADOW_REQUEST_SCHEMA = @@ -324,3 +336,755 @@ export async function recordLocalAuthorityShadow( }); } } + +// --------------------------------------------------------------------------- +// Transaction-bound entries (Stage 2C second half). +// +// A drained outbox entry becomes exactly one candidate transaction whose +// operation_id is the entry id, so a receipt names the primary transaction it +// records instead of a post-commit snapshot that may include other writers. +// --------------------------------------------------------------------------- + +export const LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1 = + "loopx_coordination_runtime_shadow_projection_v0"; +export const LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA = + "loopx_coordination_runtime_shadow_commit_entry_request_v0"; +export const LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA = + "loopx_coordination_runtime_shadow_commit_entry_result_v0"; +export const LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = + "loopx_coordination_runtime_shadow_outbox_read_v0"; +export const LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = + "loopx_coordination_runtime_shadow_outbox_read_result_v0"; +export const LOCAL_AUTHORITY_SHADOW_EVENT_SCHEMA_V1 = + "loopx_coordination_runtime_shadow_outbox_event_v0"; +export const LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA = + "loopx_coordination_runtime_shadow_outbox_receipt_v0"; + +const SHADOW_PARTITIONS = ["todos", "leases"] as const; +const ENTRY_RESOLUTIONS = [ + "committed", + "committed_proven_by_readback", + "abandoned", + "unproved", + "seed", +] as const; +const NO_OP_RESOLUTIONS = new Set(["abandoned", "unproved"]); +const SOURCE_KINDS = ["markdown_active_state", "state_event_log", "task_lease_record"] as const; +const WRITER_RUNTIMES = ["python", "typescript"] as const; +const COMMIT_ENTRY_REQUEST_FIELDS = new Set([ + "schema_version", + "runtime_root", + "goal_id", + "entry", + "partition_projection", + "partition_digest", +]); +const ENTRY_FIELDS = new Set([ + "entry_id", + "partition", + "seq", + "writer", + "source", + "source_root_digest", + "prepared_at", + "committed_at", + "resolution", +]); +const READ_REQUEST_FIELDS = new Set([ + "schema_version", + "runtime_root", + "goal_id", + "store_kind", + "scan_after_cursor", + "scan_limit", +]); +const ENTRY_ID_PATTERN = /^local-shadow-tx-[0-9a-f]{64}$/u; +const DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; +const MAX_SCAN_LIMIT = 1000; +const REVISION_RETRY_ATTEMPTS = 3; + +export type ShadowPartition = (typeof SHADOW_PARTITIONS)[number]; +export type ShadowEntryResolution = (typeof ENTRY_RESOLUTIONS)[number]; +export type LocalAuthorityShadowCommitEntryOutcome = + | "delivered" + | "replayed" + | "ambiguous_reconciled" + | "ambiguous_unproved" + | "unavailable" + | "failed" + | "protocol_mismatch" + | "conflict_retry_required"; + +interface ShadowEntryWriter { + runtime: (typeof WRITER_RUNTIMES)[number]; + write_class: string; + operation_id: string | null; +} + +interface ShadowEntrySource { + kind: (typeof SOURCE_KINDS)[number]; + previous_bytes_digest: string | null; + bytes_digest: string | null; + lease: JsonObject | null; + event_id: string | null; +} + +interface ShadowEntry { + entry_id: string; + partition: ShadowPartition; + seq: number; + writer: ShadowEntryWriter; + source: ShadowEntrySource; + source_root_digest: string; + prepared_at: string; + committed_at: string | null; + resolution: ShadowEntryResolution; +} + +interface CommitEntryRequest { + runtime_root: string; + goal_id: string; + entry: ShadowEntry; + partition_projection: JsonObject | null; + partition_digest: string | null; +} + +export interface LocalAuthorityShadowCommitEntryResult extends JsonObject { + schema_version: typeof LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA; + outcome: LocalAuthorityShadowCommitEntryOutcome; + reason_code: string | null; + goal_id: string; + entry_id: string; + partition: ShadowPartition; + seq: number; + no_op: boolean; + store_identity: string | null; + provider_revision: string | null; + cursor: string | null; + head_digest: string | null; +} + +interface ReadRequest { + runtime_root: string; + goal_id: string; + store_kind: "runtime_shadow" | "legacy_observation"; + scan_after_cursor: string | null; + scan_limit: number; +} + +function requireGoalId(value: unknown): string { + const goalId = requireNonEmptyString(value, "goal_id"); + if (goalId === "." || goalId === ".." || goalId.includes("/") || goalId.includes("\\")) { + throw new EffectRuntimeRequestError( + "Local authority shadow goal id must be a single path segment", + ); + } + return goalId; +} + +function rejectUnexpectedFields( + record: JsonObject, + allowed: Set, + label: string, +): void { + const unexpected = Object.keys(record).filter((field) => !allowed.has(field)); + if (unexpected.length > 0) { + unexpected.sort(authorityUnicodeCompare); + throw new EffectRuntimeRequestError( + `${label} has unsupported fields: ${unexpected.join(", ")}`, + ); + } +} + +function optionalString(value: unknown, label: string): string | null { + if (value === null || value === undefined) return null; + return requireNonEmptyString(value, label); +} + +function optionalDigest(value: unknown, label: string): string | null { + const digest = optionalString(value, label); + if (digest !== null && !DIGEST_PATTERN.test(digest)) { + throw new EffectRuntimeRequestError(`${label} must be sha256:<64 lowercase hex>`); + } + return digest; +} + +function decodeEntry(value: unknown): ShadowEntry { + const raw = requireJsonObject(value, "entry"); + rejectUnexpectedFields(raw, ENTRY_FIELDS, "Local authority shadow entry"); + const entryId = requireNonEmptyString(raw.entry_id, "entry.entry_id"); + if (!ENTRY_ID_PATTERN.test(entryId)) { + throw new EffectRuntimeRequestError("entry.entry_id must be local-shadow-tx-<64 lowercase hex>"); + } + const seq = requireInteger(raw.seq, "entry.seq"); + if (seq < 1) { + throw new EffectRuntimeRequestError("entry.seq must be a positive integer"); + } + const writer = requireJsonObject(raw.writer, "entry.writer"); + const source = requireJsonObject(raw.source, "entry.source"); + const lease = source.lease === null || source.lease === undefined + ? null + : requireJsonObject(source.lease, "entry.source.lease"); + return { + entry_id: entryId, + partition: requireStringLiteral(raw.partition, SHADOW_PARTITIONS, "entry.partition"), + seq, + writer: { + runtime: requireStringLiteral(writer.runtime, WRITER_RUNTIMES, "entry.writer.runtime"), + write_class: requireNonEmptyString(writer.write_class, "entry.writer.write_class"), + operation_id: optionalString(writer.operation_id, "entry.writer.operation_id"), + }, + source: { + kind: requireStringLiteral(source.kind, SOURCE_KINDS, "entry.source.kind"), + previous_bytes_digest: optionalDigest( + source.previous_bytes_digest, + "entry.source.previous_bytes_digest", + ), + bytes_digest: optionalDigest(source.bytes_digest, "entry.source.bytes_digest"), + lease: lease === null ? null : structuredClone(lease), + event_id: optionalString(source.event_id, "entry.source.event_id"), + }, + source_root_digest: requireNonEmptyString(raw.source_root_digest, "entry.source_root_digest"), + prepared_at: requireNonEmptyString(raw.prepared_at, "entry.prepared_at"), + committed_at: optionalString(raw.committed_at, "entry.committed_at"), + resolution: requireStringLiteral(raw.resolution, ENTRY_RESOLUTIONS, "entry.resolution"), + }; +} + +function decodePartitionProjection( + value: unknown, + partition: ShadowPartition, +): JsonObject | null { + if (value === null || value === undefined) return null; + const projection = requireJsonObject(value, "partition_projection"); + if (partition === "todos") { + if ( + Object.keys(projection).length !== 2 || + typeof projection.handoff_mode !== "string" || + !Array.isArray(projection.todos) + ) { + throw new EffectRuntimeRequestError( + "todos partition projection must be exactly {handoff_mode, todos[]}", + ); + } + } else if (Object.keys(projection).length !== 1 || !Array.isArray(projection.leases)) { + throw new EffectRuntimeRequestError("leases partition projection must be exactly {leases[]}"); + } + return structuredClone(projection); +} + +function decodeCommitEntryRequest(value: unknown): CommitEntryRequest { + const request = requireJsonObject(value, "local authority shadow commit entry request"); + rejectUnexpectedFields( + request, + COMMIT_ENTRY_REQUEST_FIELDS, + "Local authority shadow commit entry request", + ); + if (request.schema_version !== LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA) { + throw new EffectRuntimeRequestError( + "Local authority shadow commit entry request schema mismatch", + ); + } + const entry = decodeEntry(request.entry); + const projection = decodePartitionProjection(request.partition_projection, entry.partition); + const digest = optionalDigest(request.partition_digest, "partition_digest"); + const noOp = NO_OP_RESOLUTIONS.has(entry.resolution); + if (noOp && (projection !== null || digest !== null)) { + throw new EffectRuntimeRequestError( + `entry resolution ${entry.resolution} must not carry a partition projection`, + ); + } + if (!noOp && (projection === null || digest === null)) { + throw new EffectRuntimeRequestError( + `entry resolution ${entry.resolution} requires partition_projection and partition_digest`, + ); + } + return { + runtime_root: requireNonEmptyString(request.runtime_root, "runtime_root"), + goal_id: requireGoalId(request.goal_id), + entry, + partition_projection: projection, + partition_digest: digest, + }; +} + +function decodeReadRequest(value: unknown): ReadRequest { + const request = requireJsonObject(value, "local authority shadow read request"); + rejectUnexpectedFields(request, READ_REQUEST_FIELDS, "Local authority shadow read request"); + if (request.schema_version !== LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA) { + throw new EffectRuntimeRequestError("Local authority shadow read request schema mismatch"); + } + const limit = request.scan_limit === undefined ? 0 : requireInteger(request.scan_limit, "scan_limit"); + if (limit < 0 || limit > MAX_SCAN_LIMIT) { + throw new EffectRuntimeRequestError(`scan_limit must be between 0 and ${MAX_SCAN_LIMIT}`); + } + return { + runtime_root: requireNonEmptyString(request.runtime_root, "runtime_root"), + goal_id: requireGoalId(request.goal_id), + store_kind: request.store_kind === undefined || request.store_kind === "runtime_shadow" + ? "runtime_shadow" + : request.store_kind === "legacy_observation" + ? "legacy_observation" + : (() => { + throw new EffectRuntimeRequestError( + "Local authority shadow read store_kind must be runtime_shadow or legacy_observation", + ); + })(), + scan_after_cursor: optionalString(request.scan_after_cursor, "scan_after_cursor"), + scan_limit: limit, + }; +} + +/** Digest of the fields parity compares; must match Python `head_digest`. */ +export function localAuthorityShadowHeadDigest(head: JsonObject): string { + const view = { + handoff_mode: head.handoff_mode ?? null, + todos: head.todos ?? null, + leases: head.leases ?? null, + }; + return `sha256:${createHash("sha256").update(canonicalAuthorityBytes(view)).digest("hex")}`; +} + +function partitionsOf(head: JsonObject | null): JsonObject { + const raw = head?.partitions; + const partitions: JsonObject = { todos: null, leases: null }; + if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) { + for (const partition of SHADOW_PARTITIONS) { + const marker = (raw as JsonObject)[partition]; + if (marker !== null && typeof marker === "object" && !Array.isArray(marker)) { + partitions[partition] = structuredClone(marker); + } + } + } + return partitions; +} + +function todoReadModel(todos: readonly JsonObject[]): JsonObject { + return { + schema_version: "loopx_todo_canonical_read_record_v0", + todo_count: todos.length, + records_sha256: createHash("sha256").update(canonicalAuthorityBytes(todos)).digest("hex"), + contract_fields: [...TODO_CANONICAL_READ_RECORD_FIELDS], + }; +} + +/** + * Fold one partition into the candidate head. A v0 head (whole-snapshot + * observation) is accepted as the starting point with no partition markers. + */ +export function composeLocalAuthorityShadowHead( + current: JsonObject | null, + goalId: string, + entry: { partition: ShadowPartition; seq: number }, + projection: JsonObject | null, + digest: string | null, +): JsonObject { + const base = current ?? {}; + let handoffMode: string | null = typeof base.handoff_mode === "string" ? base.handoff_mode : null; + let todos = Array.isArray(base.todos) ? structuredClone(base.todos) : []; + let leases = Array.isArray(base.leases) ? structuredClone(base.leases) : []; + const partitions = partitionsOf(current); + if (projection !== null) { + if (entry.partition === "todos") { + handoffMode = String(projection.handoff_mode); + todos = structuredClone(projection.todos as JsonObject[]); + } else { + leases = structuredClone(projection.leases as JsonObject[]); + } + partitions[entry.partition] = { seq: entry.seq, partition_digest: digest }; + } + const next = { + schema_version: LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1, + goal_id: goalId, + source_authority: "legacy_markdown_and_task_lease", + handoff_mode: handoffMode, + todos, + leases, + todo_read_model: todoReadModel(todos), + partitions, + }; + return next; +} + +function transactionReceipt(request: CommitEntryRequest, noOp: boolean): JsonObject { + const { entry } = request; + return { + schema_version: LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA, + entry_id: entry.entry_id, + partition: entry.partition, + seq: entry.seq, + write_class: entry.writer.write_class, + writer_runtime: entry.writer.runtime, + writer_operation_id: entry.writer.operation_id, + source_kind: entry.source.kind, + source_bytes_digest: entry.source.bytes_digest, + source_previous_bytes_digest: entry.source.previous_bytes_digest, + source_event_id: entry.source.event_id, + source_lease: entry.source.lease, + source_root_digest: entry.source_root_digest, + partition_digest: request.partition_digest, + resolution: entry.resolution, + no_op: noOp, + prepared_at: entry.prepared_at, + committed_at: entry.committed_at, + drained_at: new Date().toISOString(), + source_transaction_correlated: true, + durable_source_outbox: true, + parity_verdict: "not_evaluated", + primary_authority: "legacy_local", + candidate_read_for_decision: false, + provider_to_local_writes: false, + }; +} + +function transactionEvent(request: CommitEntryRequest, noOp: boolean): JsonObject { + const { entry } = request; + let kind = "source_transaction_delivered"; + if (entry.resolution === "seed") kind = "partition_seeded"; + else if (entry.resolution === "abandoned") kind = "source_transaction_abandoned"; + else if (entry.resolution === "unproved") kind = "source_transaction_unproved"; + return { + schema_version: LOCAL_AUTHORITY_SHADOW_EVENT_SCHEMA_V1, + kind, + partition: entry.partition, + seq: entry.seq, + entry_id: entry.entry_id, + write_class: entry.writer.write_class, + partition_digest: request.partition_digest, + no_op: noOp, + }; +} + +function commitEntryResult( + request: CommitEntryRequest, + outcome: LocalAuthorityShadowCommitEntryOutcome, + options: { + reasonCode?: string | null; + storeIdentity?: string | null; + providerRevision?: string | null; + cursor?: string | null; + headDigest?: string | null; + } = {}, +): LocalAuthorityShadowCommitEntryResult { + return { + schema_version: LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA, + outcome, + reason_code: options.reasonCode ?? null, + goal_id: request.goal_id, + entry_id: request.entry.entry_id, + partition: request.entry.partition, + seq: request.entry.seq, + no_op: NO_OP_RESOLUTIONS.has(request.entry.resolution), + store_identity: options.storeIdentity ?? null, + provider_revision: options.providerRevision ?? null, + cursor: options.cursor ?? null, + head_digest: options.headDigest ?? null, + }; +} + +function transactionReceiptMatches( + request: CommitEntryRequest, + result: Extract, +): boolean { + return result.receipts.some((raw) => { + const receipt = raw as Record; + return receipt.schema_version === LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA && + receipt.entry_id === request.entry.entry_id && + receipt.partition === request.entry.partition && + receipt.seq === request.entry.seq && + (receipt.partition_digest ?? null) === request.partition_digest && + receipt.primary_authority === "legacy_local" && + receipt.provider_to_local_writes === false; + }); +} + +async function reconcileTransactionReceipt( + store: AuthorityStore, + request: CommitEntryRequest, + storeIdentity: string, + reconciledOutcome: "replayed" | "ambiguous_reconciled", +): Promise { + const result = await store.readReceipt(request.entry.entry_id); + if (result.status === "found" && transactionReceiptMatches(request, result)) { + return commitEntryResult(request, reconciledOutcome, { + storeIdentity, + providerRevision: result.provider_revision, + cursor: result.cursor, + }); + } + if (result.status === "unavailable" || result.status === "failed") { + return commitEntryResult(request, result.status, { + reasonCode: result.reason_code, + storeIdentity, + }); + } + return commitEntryResult( + request, + reconciledOutcome === "ambiguous_reconciled" ? "ambiguous_unproved" : "protocol_mismatch", + { + reasonCode: result.status === "missing" + ? "transaction_receipt_missing" + : "transaction_receipt_mismatch", + storeIdentity, + }, + ); +} + +function openShadowStore( + runtimeRoot: string, + goalId: string, + storeKind: ReadRequest["store_kind"], + dependencies: LocalAuthorityShadowDependencies, +): AuthorityStore { + const providerDirectory = storeKind === "legacy_observation" + ? join(runtimeRoot, "authority-shadow", "file", goalId) + : join(runtimeRoot, "authority-shadow", "file-v0"); + return (dependencies.openStore ?? ((directory, id) => new FileAuthorityStore(directory, id)))( + providerDirectory, + goalId, + ); +} + +type CommitAttempt = + | { kind: "final"; result: LocalAuthorityShadowCommitEntryResult } + | { kind: "retry"; result: LocalAuthorityShadowCommitEntryResult }; + +async function settleCommitOutcome( + store: AuthorityStore, + request: CommitEntryRequest, + storeIdentity: string, + committed: AuthorityStoreCommitResult, + headDigest: string, +): Promise { + if (committed.status === "applied") { + return { + kind: "final", + result: commitEntryResult(request, "delivered", { + storeIdentity, + providerRevision: committed.provider_revision, + cursor: committed.cursor, + headDigest, + }), + }; + } + if (committed.status === "ambiguous") { + return { + kind: "final", + result: await reconcileTransactionReceipt(store, request, storeIdentity, "ambiguous_reconciled"), + }; + } + if (committed.status === "failed") { + return { + kind: "final", + result: commitEntryResult(request, "failed", { + reasonCode: committed.reason_code, + storeIdentity, + }), + }; + } + if (committed.conflict_kind === "operation_id_exists") { + return { + kind: "final", + result: await reconcileTransactionReceipt(store, request, storeIdentity, "replayed"), + }; + } + return { + kind: "retry", + result: commitEntryResult(request, "conflict_retry_required", { + reasonCode: "provider_revision_mismatch", + storeIdentity, + providerRevision: committed.current_provider_revision, + cursor: committed.current_cursor, + }), + }; +} + +/** One load-compose-commit attempt against the current provider revision. */ +async function attemptCommitEntry( + store: AuthorityStore, + request: CommitEntryRequest, + storeIdentity: string, + noOp: boolean, +): Promise { + const loaded = await store.loadAuthority(); + if (loaded.status === "unavailable" || loaded.status === "failed") { + return { + kind: "final", + result: commitEntryResult(request, loaded.status, { + reasonCode: loaded.reason_code, + storeIdentity, + }), + }; + } + const nextHead = composeLocalAuthorityShadowHead( + loaded.status === "loaded" ? loaded.head : null, + request.goal_id, + request.entry, + request.partition_projection, + request.partition_digest, + ); + const committed = await store.commitAuthority({ + expected_provider_revision: loaded.status === "loaded" ? loaded.provider_revision : null, + operation_id: request.entry.entry_id, + events: [transactionEvent(request, noOp)], + next_projection: nextHead, + receipts: [transactionReceipt(request, noOp)], + }); + return await settleCommitOutcome( + store, + request, + storeIdentity, + committed, + localAuthorityShadowHeadDigest(nextHead), + ); +} + +/** + * Commit one drained outbox entry as exactly one candidate transaction. + * + * `operation_id` is the entry id, so a retry after a lost response replays + * onto the same transaction instead of recording the source write twice. + * No-op resolutions (abandoned / unproved) keep the sequence chain auditable + * without changing the compared head fields. + */ +export async function commitLocalAuthorityShadowEntry( + value: unknown, + dependencies: LocalAuthorityShadowDependencies = {}, +): Promise { + const request = decodeCommitEntryRequest(value); + const noOp = NO_OP_RESOLUTIONS.has(request.entry.resolution); + let store: AuthorityStore; + try { + store = openShadowStore( + request.runtime_root, + request.goal_id, + "runtime_shadow", + dependencies, + ); + } catch { + return commitEntryResult(request, "unavailable", { + reasonCode: "provider_construction_failed", + }); + } + try { + const identity = await store.storeIdentity(); + if (identity.status !== "available") { + return commitEntryResult(request, identity.status, { reasonCode: identity.reason_code }); + } + let attempt: CommitAttempt | null = null; + for (let index = 0; index < REVISION_RETRY_ATTEMPTS; index += 1) { + attempt = await attemptCommitEntry(store, request, identity.store_identity, noOp); + if (attempt.kind === "final") return attempt.result; + } + return (attempt as CommitAttempt).result; + } catch { + return commitEntryResult(request, "unavailable", { reasonCode: "provider_call_failed" }); + } +} + +function readResultBase(goalId: string): JsonObject { + return { + schema_version: LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA, + goal_id: goalId, + status: "unavailable", + reason_code: null, + store_identity: null, + provider_revision: null, + cursor: null, + head: null, + head_digest: null, + partitions: null, + scan: null, + }; +} + +function loadedReadResult( + base: JsonObject, + storeIdentity: string, + loaded: Extract, +): JsonObject { + const result: JsonObject = { ...base, status: loaded.status, store_identity: storeIdentity }; + if (loaded.status === "loaded") { + result.provider_revision = loaded.provider_revision; + result.cursor = loaded.cursor; + result.head = structuredClone(loaded.head); + result.head_digest = localAuthorityShadowHeadDigest(loaded.head); + result.partitions = partitionsOf(loaded.head); + } + return result; +} + +/** One committed transaction with its projection reduced to a digest. */ +function scanTransactionView(transaction: AuthorityStoreCommittedTransaction): JsonObject { + return { + cursor: transaction.cursor, + provider_revision: transaction.provider_revision, + operation_id: transaction.operation_id, + projection_digest: localAuthorityShadowHeadDigest(transaction.projection), + projection_partitions: partitionsOf(transaction.projection), + events: structuredClone(transaction.events) as JsonObject[], + receipts: structuredClone(transaction.receipts) as JsonObject[], + }; +} + +async function appendScanPage( + store: AuthorityStore, + request: ReadRequest, + result: JsonObject, +): Promise { + const page = await store.scanCommitted(request.scan_after_cursor, request.scan_limit); + if (page.status !== "page") { + return { ...result, status: page.status, reason_code: page.reason_code }; + } + return { + ...result, + scan: { + transactions: page.transactions.map(scanTransactionView), + next_cursor: page.next_cursor, + has_more: page.has_more, + }, + }; +} + +/** + * Read-only view of the candidate store for drain readback and parity: + * head, its comparison digest, and a page of committed transactions with the + * projection reduced to its digest so responses stay bounded. + */ +export async function readLocalAuthorityShadow( + value: unknown, + dependencies: LocalAuthorityShadowDependencies = {}, +): Promise { + const request = decodeReadRequest(value); + const base = readResultBase(request.goal_id); + let store: AuthorityStore; + try { + store = openShadowStore( + request.runtime_root, + request.goal_id, + request.store_kind, + dependencies, + ); + } catch { + return { ...base, reason_code: "provider_construction_failed" }; + } + try { + const identity = await store.storeIdentity(); + if (identity.status !== "available") { + return { ...base, status: identity.status, reason_code: identity.reason_code }; + } + const loaded = await store.loadAuthority(); + if (loaded.status === "unavailable" || loaded.status === "failed") { + return { + ...base, + status: loaded.status, + reason_code: loaded.reason_code, + store_identity: identity.store_identity, + }; + } + const result = loadedReadResult(base, identity.store_identity, loaded); + return request.scan_limit > 0 ? await appendScanPage(store, request, result) : result; + } catch { + return { ...base, reason_code: "provider_call_failed" }; + } +} diff --git a/loopx/control_plane/coordination/local_authority_shadow_adapter.py b/loopx/control_plane/coordination/local_authority_shadow_adapter.py index fed81cb32f..1bed5207a3 100644 --- a/loopx/control_plane/coordination/local_authority_shadow_adapter.py +++ b/loopx/control_plane/coordination/local_authority_shadow_adapter.py @@ -10,15 +10,35 @@ import hashlib import json -from collections.abc import Mapping +import time +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any -from ...file_lock import LockAcquireTimeoutError, exclusive_file_lock +from ...file_lock import ( + LockAcquireTimeoutError, + exclusive_cross_runtime_file_lock, + exclusive_file_lock, + try_exclusive_file_lock, +) from ...history import load_registry from ...paths import resolve_runtime_root from ...registry import find_registry_goal from ..effect_runtime import effect_runtime_result +from . import local_authority_shadow_outbox as outbox +from .local_authority_shadow_projection import ( + LEASE_PARTITION, + PARTITIONS, + TODO_PARTITION, + canonical_value, + head_digest, + partition_digest, + text_digest, + todo_partition_projection, +) +from .runtime_shadow import resolve_coordination_runtime_shadow_config LOCAL_AUTHORITY_SHADOW_CONFIG_SCHEMA = "loopx_local_authority_shadow_config_v0" @@ -489,3 +509,985 @@ def observe_todo_local_authority_commit( "observe_todo_local_authority_commit", "validate_local_authority_shadow_change", ] + + +# --------------------------------------------------------------------------- +# Transaction-bound outbox drain (Stage 2C second half plumbing). +# +# Writers record per-partition outbox entries inside the primary lock (see +# ``local_authority_shadow_outbox``). The drain below runs after that lock is +# released and turns each committed entry into exactly one candidate +# transaction. It is bounded, never blocks a writer, and reports what it left +# behind instead of guessing. +# --------------------------------------------------------------------------- + +LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1 = "loopx_local_authority_shadow_evidence_v1" +LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA = ( + "loopx_coordination_runtime_shadow_commit_entry_request_v0" +) +LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA = ( + "loopx_coordination_runtime_shadow_commit_entry_result_v0" +) +LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA = "loopx_coordination_runtime_shadow_outbox_read_v0" +LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA = "loopx_coordination_runtime_shadow_outbox_read_result_v0" +INLINE_DRAIN_MAX_ENTRIES = 16 +INLINE_DRAIN_BUDGET_SECONDS = 2.0 +INLINE_DRAIN_LOCK_TIMEOUT_SECONDS = 0.25 +CLI_DRAIN_LOCK_TIMEOUT_SECONDS = 5.0 +RETENTION_PRESSURE_BYTES = 8 * 1024 * 1024 +_COMMIT_ENTRY_OUTCOMES = { + "delivered", + "replayed", + "ambiguous_reconciled", + "ambiguous_unproved", + "unavailable", + "failed", + "protocol_mismatch", + "conflict_retry_required", +} +_SETTLED_OUTCOMES = {"delivered", "replayed", "ambiguous_reconciled"} +_SEED_WRITE_CLASSES = {"seed", "reseed_after_crash_gap"} +_ENTRY_SOURCE_FIELDS = ("kind", "previous_bytes_digest", "bytes_digest", "lease", "event_id") +_EVIDENCE_V1_OUTCOMES = { + "delivered", + "replayed", + "ambiguous_reconciled", + "pending", + "drain_deferred", + "no_transaction", + "capture_failed", + "ambiguous_unproved", + "unavailable", + "failed", + "protocol_mismatch", + "conflict_retry_required", +} + + +@dataclass +class DrainResult: + """Typed outcome of one bounded drain pass.""" + + goal_id: str + outcome: str = "nothing_pending" + config_enabled: bool = False + delivered: int = 0 + replayed: int = 0 + reconciled: int = 0 + no_op: int = 0 + reseeded: int = 0 + reclaimed_residue: int = 0 + pending_after: int = 0 + prepared_only_after: int = 0 + in_flight_partitions: list[str] = field(default_factory=list) + budget_exhausted: bool = False + stopped_at: dict[str, Any] | None = None + reason_code: str | None = None + store_identity: str | None = None + provider_revision: str | None = None + last_cursor: str | None = None + cursor_before: str | None = None + cursor_after: str | None = None + head_digest: str | None = None + candidate_readback_verified: bool | None = None + entries: list[dict[str, Any]] = field(default_factory=list) + + @property + def ok(self) -> bool: + return self.outcome in {"drained", "nothing_pending"} and self.stopped_at is None + + @property + def drained_count(self) -> int: + return self.delivered + self.replayed + self.reconciled + + def entry_outcome(self, entry_id: str | None) -> dict[str, Any] | None: + if entry_id is None: + return None + for item in self.entries: + if item.get("entry_id") == entry_id: + return item + return None + + def to_payload(self) -> dict[str, Any]: + payload = asdict(self) + payload["ok"] = self.ok + payload["drained_count"] = self.drained_count + return payload + + +def todo_partition_projector( + goal: Mapping[str, Any] | None, + *, + state_path: Path, + rollout_events: list[dict[str, Any]] | None = None, +) -> outbox.TodoPartitionProjector: + """Production projector: parse active-state text into the todos partition.""" + + from ...control_plane.todos.handoff_mode import goal_handoff_mode + from ..todos.goal_todo_projection import project_goal_todo_items + + goal_record = dict(goal) if isinstance(goal, Mapping) else None + events = list(rollout_events or []) + + def project(state_text: str) -> dict[str, Any]: + return todo_partition_projection( + handoff_mode=goal_handoff_mode(state_text), + todos=project_goal_todo_items( + goal_record, + state_text=state_text, + state_path=state_path, + rollout_events=events, + ), + ) + + return project + + +def primary_lock_is_free(target: Path) -> bool: + """Probe a partition's Python primary lock once without waiting.""" + + try: + with try_exclusive_file_lock(target, operation="local_authority_shadow_drain_probe") as held: + return held is not None + except OSError: + return False + + +@dataclass(frozen=True) +class _GoalSources: + goal: dict[str, Any] | None + state_path: Path + lease_dir: Path + + +def _goal_sources( + registry: dict[str, Any], + *, + runtime_root: Path, + goal_id: str, +) -> _GoalSources: + from ...state_refresh import resolve_goal_state + + goal, _project, state_path = resolve_goal_state( + registry=registry, + goal_id=goal_id, + project_override=None, + state_file_override=None, + ) + return _GoalSources( + goal=goal, + state_path=state_path, + lease_dir=outbox.lease_directory(runtime_root, goal_id), + ) + + +def _read_state_text(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def _event_present(sources: _GoalSources, event_id: str) -> bool: + from ...event_sourced_state import AppendOnlyStateEventStore + from ...status import state_event_log_candidates + + if sources.goal is None: + return False + for candidate in state_event_log_candidates(sources.goal, state_path=sources.state_path): + if not candidate.exists(): + continue + for event in AppendOnlyStateEventStore(candidate).load(): + if isinstance(event, dict) and event.get("event_id") == event_id: + return True + return False + + +def _lease_record(sources: _GoalSources, todo_id: str) -> dict[str, Any] | None: + if not todo_id: + return None + path = sources.lease_dir / f"{todo_id}.json" + if not path.exists(): + return None + raw = json.loads(path.read_text(encoding="utf-8")) + return raw if isinstance(raw, dict) else None + + +def _todo_partition_seed( + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + sources: _GoalSources, +) -> outbox.SeedSource: + """Full todos-partition snapshot; caller holds the state-file lock.""" + + from ...control_plane.todos.handoff_mode import goal_handoff_mode + from ...todos import list_goal_todos + + state_text = _read_state_text(sources.state_path) + payload = list_goal_todos( + registry_path=registry_path, + goal_id=goal_id, + runtime_root_arg=str(runtime_root), + ) + projection = todo_partition_projection( + handoff_mode=goal_handoff_mode(state_text), + todos=payload.get("todos") or [], + ) + return outbox.SeedSource( + partition=TODO_PARTITION, + projection=projection, + source_bytes_digest=text_digest(state_text), + ) + + +@contextmanager +def _primary_lock_if_free( + partition: str, + *, + runtime_root: Path, + goal_id: str, + sources: _GoalSources, +) -> Iterator[bool]: + """Hold the partition's primary lock only if it is free right now.""" + + if partition == TODO_PARTITION: + with try_exclusive_file_lock( + sources.state_path, + operation="local_authority_shadow_drain_resolve", + ) as held: + yield held is not None + return + from ..work_items.task_lease import task_lease_lock_path + + target = task_lease_lock_path(runtime_root=runtime_root, goal_id=goal_id) + try: + with exclusive_cross_runtime_file_lock( + target, + timeout_seconds=0.0, + operation="local_authority_shadow_drain_resolve", + ): + yield True + except LockAcquireTimeoutError: + yield False + + +def _entry_projection( + entry: outbox.OutboxEntry, + *, + goal_id: str, +) -> tuple[dict[str, Any] | None, str | None]: + """Compact projection and digest for a deliverable entry.""" + + raw = entry.projection() + if raw is None: + return None, None + if entry.partition == LEASE_PARTITION: + projection = outbox.compact_lease_projection(raw, goal_id=goal_id) + return projection, partition_digest(projection) + projection = dict(canonical_value(raw)) + digest = partition_digest(projection) + recorded = entry.recorded_partition_digest() + if recorded is not None and recorded != digest: + raise outbox.OutboxError( + "outbox_file_invalid", + f"entry {entry.entry_id} projection does not match its recorded digest", + ) + return projection, digest + + +def _commit_entry_request( + *, + runtime_root: Path, + goal_id: str, + entry: outbox.OutboxEntry, + resolution: str, + projection: dict[str, Any] | None, + digest: str | None, +) -> dict[str, Any]: + raw_source = entry.prepared.get("source") if isinstance(entry.prepared.get("source"), dict) else {} + writer = entry.prepared.get("writer") if isinstance(entry.prepared.get("writer"), dict) else {} + committed_at = entry.committed.get("committed_at") if entry.committed else None + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA, + "runtime_root": str(runtime_root), + "goal_id": goal_id, + "entry": { + "entry_id": entry.entry_id, + "partition": entry.partition, + "seq": entry.seq, + "writer": { + "runtime": writer.get("runtime"), + "write_class": writer.get("write_class"), + "operation_id": writer.get("operation_id"), + }, + "source": {key: raw_source.get(key) for key in _ENTRY_SOURCE_FIELDS}, + "source_root_digest": entry.prepared.get("source_root_digest"), + "prepared_at": entry.prepared.get("prepared_at"), + "committed_at": committed_at, + "resolution": resolution, + }, + "partition_projection": projection, + "partition_digest": digest, + } + + +def _valid_commit_entry_result(result: object, entry: outbox.OutboxEntry) -> bool: + if not isinstance(result, dict): + return False + return ( + result.get("schema_version") == LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA + and result.get("outcome") in _COMMIT_ENTRY_OUTCOMES + and result.get("entry_id") == entry.entry_id + and result.get("partition") == entry.partition + and result.get("seq") == entry.seq + and isinstance(result.get("no_op"), bool) + ) + + +def read_local_authority_shadow( + *, + runtime_root: Path, + goal_id: str, + store_kind: str = "runtime_shadow", + scan_after_cursor: str | None = None, + scan_limit: int = 0, +) -> dict[str, Any]: + """Read-only candidate view through the TypeScript store boundary.""" + + result = effect_runtime_result( + "coordination.runtime_shadow.outbox_read", + { + "schema_version": LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA, + "runtime_root": str(runtime_root), + "goal_id": goal_id, + "store_kind": store_kind, + "scan_after_cursor": scan_after_cursor, + "scan_limit": scan_limit, + }, + timeout=15.0, + ) + if ( + not isinstance(result, dict) + or result.get("schema_version") != LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA + or result.get("goal_id") != goal_id + ): + raise RuntimeError("local authority shadow read result is invalid") + return dict(result) + + +class _DrainBudget: + def __init__(self, *, max_entries: int, budget_seconds: float) -> None: + self._max_entries = max(1, max_entries) + self._deadline = time.monotonic() + max(0.0, budget_seconds) + self.consumed = 0 + + def exhausted(self) -> bool: + return self.consumed >= self._max_entries or time.monotonic() >= self._deadline + + +class _PartitionDrainer: + """Drain one partition in sequence order; all state lives on ``result``.""" + + def __init__( + self, + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + partition: str, + sources: _GoalSources, + result: DrainResult, + budget: _DrainBudget, + ) -> None: + self._registry_path = registry_path + self._runtime_root = runtime_root + self._goal_id = goal_id + self._partition = partition + self._sources = sources + self._result = result + self._budget = budget + self._directory = outbox.partition_directory(runtime_root, goal_id, partition) + self.last_delivered_digest: str | None = None + + def run(self) -> None: + # Files the cursor already covers are settled; reclaim them first so a + # crash between the cursor write and the unlinks can never wedge the + # partition. + self._result.reclaimed_residue += outbox.reclaim_retired_residue(self._directory) + while not self._budget.exhausted(): + entries = outbox.list_entries(self._directory) + if not entries: + return + entry = entries[0] + if entry.is_committed: + settled = self._deliver_committed(entry) + else: + settled = self._resolve_prepared_only(entry) + if not settled: + return + if outbox.list_entries(self._directory): + self._result.budget_exhausted = True + + def _deliver_committed(self, entry: outbox.OutboxEntry) -> bool: + writer = entry.prepared.get("writer") if isinstance(entry.prepared.get("writer"), dict) else {} + resolution = "seed" if writer.get("write_class") in _SEED_WRITE_CLASSES else "committed" + projection, digest = _entry_projection(entry, goal_id=self._goal_id) + if projection is None: + raise outbox.OutboxError( + "outbox_file_invalid", + f"committed entry {entry.entry_id} has no partition projection", + ) + return self._commit(entry, resolution=resolution, projection=projection, digest=digest) + + def _resolve_prepared_only(self, entry: outbox.OutboxEntry) -> bool: + with _primary_lock_if_free( + self._partition, + runtime_root=self._runtime_root, + goal_id=self._goal_id, + sources=self._sources, + ) as held: + if not held: + if self._partition not in self._result.in_flight_partitions: + self._result.in_flight_partitions.append(self._partition) + return False + resolution = outbox.resolve_prepared_only_entry( + entry, + markdown_text_reader=lambda: _read_state_text(self._sources.state_path), + lease_record_reader=lambda todo_id: _lease_record(self._sources, todo_id), + event_presence_reader=lambda event_id: _event_present(self._sources, event_id), + ) + projection: dict[str, Any] | None = None + digest: str | None = None + if resolution == "committed": + projection, digest = _entry_projection(entry, goal_id=self._goal_id) + if projection is None: + resolution = "unproved" + else: + resolution = "committed_proven_by_readback" + if resolution == "unproved": + # The source moved in a way no recorded entry explains; a full + # partition snapshot under the same lock closes the gap. + seed = ( + _todo_partition_seed( + registry_path=self._registry_path, + runtime_root=self._runtime_root, + goal_id=self._goal_id, + sources=self._sources, + ) + if self._partition == TODO_PARTITION + else outbox.lease_seed_source(self._runtime_root, self._goal_id) + ) + outbox.write_seed_entry( + runtime_root=self._runtime_root, + goal_id=self._goal_id, + seed=seed, + write_class="reseed_after_crash_gap", + ) + self._result.reseeded += 1 + return self._commit(entry, resolution=resolution, projection=projection, digest=digest) + + def _commit( + self, + entry: outbox.OutboxEntry, + *, + resolution: str, + projection: dict[str, Any] | None, + digest: str | None, + ) -> bool: + expected_root = outbox.runtime_root_digest(self._runtime_root) + if entry.prepared.get("source_root_digest") != expected_root: + # The entry was written for a different runtime root; delivering it + # here would stitch another lineage's transaction into this one. + raise outbox.OutboxError( + "source_root_mismatch", + f"entry {entry.entry_id} was recorded for a different runtime root", + ) + request = _commit_entry_request( + runtime_root=self._runtime_root, + goal_id=self._goal_id, + entry=entry, + resolution=resolution, + projection=projection, + digest=digest, + ) + raw = effect_runtime_result( + "coordination.runtime_shadow.commit_entry", + request, + timeout=15.0, + ) + self._budget.consumed += 1 + if not _valid_commit_entry_result(raw, entry): + self._result.stopped_at = { + "partition": entry.partition, + "seq": entry.seq, + "entry_id": entry.entry_id, + "outcome": "failed", + "reason_code": "shadow_commit_entry_result_invalid", + } + return False + result = dict(raw) + summary = { + "entry_id": entry.entry_id, + "partition": entry.partition, + "seq": entry.seq, + "resolution": resolution, + "outcome": result["outcome"], + "reason_code": result.get("reason_code"), + "cursor": result.get("cursor"), + "provider_revision": result.get("provider_revision"), + "partition_digest": digest, + } + self._result.entries.append(summary) + if result.get("store_identity"): + self._result.store_identity = str(result["store_identity"]) + if result["outcome"] not in _SETTLED_OUTCOMES: + self._result.stopped_at = { + "partition": entry.partition, + "seq": entry.seq, + "entry_id": entry.entry_id, + "outcome": result["outcome"], + "reason_code": result.get("reason_code"), + } + return False + previous = outbox.read_cursor(self._directory) + cursor_digest = digest + if cursor_digest is None and previous is not None: + cursor_digest = previous.get("last_partition_digest") + outbox.write_cursor( + self._directory, + partition=entry.partition, + last_seq=entry.seq, + last_entry_id=entry.entry_id, + last_partition_digest=cursor_digest, + last_cursor=result.get("cursor"), + last_provider_revision=result.get("provider_revision"), + ) + outbox.remove_entry_files(entry) + if result["outcome"] == "delivered": + self._result.delivered += 1 + elif result["outcome"] == "replayed": + self._result.replayed += 1 + else: + self._result.reconciled += 1 + if result["no_op"]: + self._result.no_op += 1 + elif digest is not None: + self.last_delivered_digest = digest + if result.get("cursor"): + self._result.last_cursor = str(result["cursor"]) + if result.get("provider_revision"): + self._result.provider_revision = str(result["provider_revision"]) + return True + + +def _candidate_cursor(runtime_root: Path, goal_id: str) -> str | None: + """Current candidate cursor, or None when the store has no document yet.""" + + directory = runtime_root / "authority-shadow" / "file-v0" + if not directory.is_dir() or not any(directory.glob("authority-store-*.json")): + return None + try: + view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) + except Exception: + return None + cursor = view.get("cursor") + return str(cursor) if isinstance(cursor, str) else None + + +def _verify_readback( + result: DrainResult, + *, + runtime_root: Path, + goal_id: str, + delivered_digests: dict[str, str], +) -> None: + try: + view = read_local_authority_shadow(runtime_root=runtime_root, goal_id=goal_id) + except Exception: + result.candidate_readback_verified = False + return + head = view.get("head") + if view.get("status") != "loaded" or not isinstance(head, dict): + result.candidate_readback_verified = False + return + result.store_identity = view.get("store_identity") or result.store_identity + result.head_digest = view.get("head_digest") + result.cursor_after = view.get("cursor") + verified = head_digest(head) == view.get("head_digest") + partitions = head.get("partitions") if isinstance(head.get("partitions"), dict) else {} + for partition, digest in delivered_digests.items(): + marker = partitions.get(partition) if isinstance(partitions, dict) else None + verified = verified and isinstance(marker, dict) and marker.get("partition_digest") == digest + result.candidate_readback_verified = verified + + +def _drain_prelude( + result: DrainResult, + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, +) -> tuple[dict[str, Any], Path] | None: + """Validate the goal id and resolve the registry and root; typed failure on error.""" + + if not goal_id or goal_id in {".", ".."} or "/" in goal_id or "\\" in goal_id: + result.outcome = "failed" + result.reason_code = "invalid_shadow_goal_id" + return None + try: + registry = load_registry(registry_path) + result.config_enabled = ( + resolve_coordination_runtime_shadow_config( + find_registry_goal(registry, goal_id) + ).enabled + or _shadow_config(registry, goal_id) is not None + ) + resolved = ( + runtime_root + if runtime_root is not None + else resolve_runtime_root(registry, None, registry_path=registry_path) + ) + except Exception: + result.outcome = "failed" + result.reason_code = "invalid_shadow_config" + return None + return registry, resolved + + +def _outbox_is_idle(summary: Mapping[str, Mapping[str, Any]]) -> bool: + return all( + item["committed_pending"] == 0 + and item["prepared_only"] == 0 + and item["retired_residue"] == 0 + and item["invalid"] is None + for item in summary.values() + ) + + +def _drain_partitions( + result: DrainResult, + *, + registry: dict[str, Any], + registry_path: Path, + runtime_root: Path, + goal_id: str, + max_entries: int, + budget_seconds: float, +) -> None: + """Drain every partition in order under the held drain lock, then read back.""" + + sources = _goal_sources(registry, runtime_root=runtime_root, goal_id=goal_id) + result.cursor_before = _candidate_cursor(runtime_root, goal_id) + budget = _DrainBudget(max_entries=max_entries, budget_seconds=budget_seconds) + delivered_digests: dict[str, str] = {} + for partition in PARTITIONS: + if result.stopped_at is not None: + break + drainer = _PartitionDrainer( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + partition=partition, + sources=sources, + result=result, + budget=budget, + ) + drainer.run() + if drainer.last_delivered_digest is not None: + delivered_digests[partition] = drainer.last_delivered_digest + if delivered_digests or result.drained_count: + _verify_readback( + result, + runtime_root=runtime_root, + goal_id=goal_id, + delivered_digests=delivered_digests, + ) + + +def _settle_drain_outcome(result: DrainResult) -> None: + if result.stopped_at is not None: + result.outcome = "stopped" + result.reason_code = str(result.stopped_at.get("reason_code") or result.stopped_at["outcome"]) + else: + result.outcome = "drained" + + +def _count_backlog(result: DrainResult, runtime_root: Path, goal_id: str) -> None: + summary_after = outbox.outbox_summary(runtime_root, goal_id) + result.pending_after = sum(int(item["committed_pending"]) for item in summary_after.values()) + result.prepared_only_after = sum(int(item["prepared_only"]) for item in summary_after.values()) + + +def drain_local_authority_shadow_outbox( + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, + max_entries: int = INLINE_DRAIN_MAX_ENTRIES, + budget_seconds: float = INLINE_DRAIN_BUDGET_SECONDS, + lock_timeout_seconds: float = INLINE_DRAIN_LOCK_TIMEOUT_SECONDS, +) -> DrainResult: + """Deliver pending outbox entries to the candidate store, one transaction each. + + The drain lock is per goal. A held lock means another drainer is already + at work, so the caller's write stays ``pending`` instead of waiting on it. + """ + + result = DrainResult(goal_id=goal_id) + prelude = _drain_prelude( + result, registry_path=registry_path, runtime_root=runtime_root, goal_id=goal_id + ) + if prelude is None: + return result + registry, resolved_root = prelude + if _outbox_is_idle(outbox.outbox_summary(resolved_root, goal_id)): + result.outcome = "nothing_pending" + return result + try: + with exclusive_file_lock( + outbox.drain_lock_target(resolved_root, goal_id), + timeout_seconds=lock_timeout_seconds, + operation="local_authority_shadow_drain", + ): + _drain_partitions( + result, + registry=registry, + registry_path=registry_path, + runtime_root=resolved_root, + goal_id=goal_id, + max_entries=max_entries, + budget_seconds=budget_seconds, + ) + except LockAcquireTimeoutError: + result.outcome = "drain_deferred" + result.reason_code = "drain_lock_busy" + except outbox.OutboxError as error: + result.outcome = "stopped" + result.reason_code = error.reason_code + except Exception: + result.outcome = "stopped" + result.reason_code = "shadow_drain_failed" + else: + _settle_drain_outcome(result) + _count_backlog(result, resolved_root, goal_id) + return result + + +class _CandidateMissing(Exception): + """The candidate store directory does not exist yet.""" + + +def _store_bytes(runtime_root: Path, goal_id: str, *, legacy_observation: bool) -> int: + directory = ( + runtime_root / "authority-shadow" / "file" / goal_id + if legacy_observation + else runtime_root / "authority-shadow" / "file-v0" + ) + if not directory.is_dir(): + return 0 + return sum(path.stat().st_size for path in directory.iterdir() if path.is_file()) + + +def local_authority_shadow_status( + *, + registry_path: Path, + runtime_root: Path | None, + goal_id: str, +) -> dict[str, Any]: + """Operator readback: configuration, outbox backlog, and candidate head facts.""" + + registry = load_registry(registry_path) + goal = find_registry_goal(registry, goal_id) + if not isinstance(goal, dict): + raise ValueError(f"goal {goal_id!r} is not registered") + if runtime_root is None: + runtime_root = resolve_runtime_root(registry, None, registry_path=registry_path) + config = local_authority_shadow_summary(goal) + legacy_observation = config["enabled"] is True + backlog = outbox.outbox_summary(runtime_root, goal_id) + candidate: dict[str, Any] + try: + directory = ( + runtime_root / "authority-shadow" / "file" / goal_id + if legacy_observation + else runtime_root / "authority-shadow" / "file-v0" + ) + if not directory.is_dir() or not any(directory.glob("authority-store-*.json")): + # Reading through the store boundary would mint a store identity; + # a status probe must not create candidate lineage. + raise _CandidateMissing + view = read_local_authority_shadow( + runtime_root=runtime_root, + goal_id=goal_id, + store_kind=("legacy_observation" if legacy_observation else "runtime_shadow"), + ) + head = view.get("head") if isinstance(view.get("head"), dict) else None + candidate = { + "status": view.get("status"), + "reason_code": view.get("reason_code"), + "store_identity": view.get("store_identity"), + "provider_revision": view.get("provider_revision"), + "cursor": view.get("cursor"), + "head_digest": view.get("head_digest"), + "head_schema_version": head.get("schema_version") if head else None, + "partitions": view.get("partitions"), + "codec_agreement": (head_digest(head) == view.get("head_digest")) if head else None, + } + except _CandidateMissing: + candidate = { + "status": "missing", + "reason_code": None, + "store_identity": None, + "provider_revision": None, + "cursor": None, + "head_digest": None, + "head_schema_version": None, + "partitions": None, + "codec_agreement": None, + } + except Exception: + candidate = { + "status": "unavailable", + "reason_code": "shadow_read_failed", + "store_identity": None, + "provider_revision": None, + "cursor": None, + "head_digest": None, + "head_schema_version": None, + "partitions": None, + "codec_agreement": None, + } + store_bytes = _store_bytes( + runtime_root, + goal_id, + legacy_observation=legacy_observation, + ) + return { + "ok": all(item["invalid"] is None for item in backlog.values()), + "action": "status", + "goal_id": goal_id, + "config": config, + "runtime_root_digest": outbox.runtime_root_digest(runtime_root), + "outbox": backlog, + "candidate": candidate, + "store_bytes": store_bytes, + "retention_pressure": store_bytes > RETENTION_PRESSURE_BYTES, + } + + +def capture_evidence( + *, + goal_id: str, + capture: outbox.CaptureOutcome, + drain: DrainResult | None, +) -> dict[str, Any]: + """Evidence v1 attached to a writer payload: capture facts plus drain facts. + + Every flag here is a measured fact of this write. ``source_candidate_compared`` + and ``parity_verdict`` stay negative until the verify step exists. + """ + + if capture.failure is not None: + outcome = "capture_failed" + reason_code: str | None = str(capture.failure.get("reason_code")) + elif capture.entry_id is None: + outcome = "no_transaction" + reason_code = capture.skipped_reason + elif drain is None or drain.outcome == "drain_deferred": + outcome = "drain_deferred" if drain is not None else "pending" + reason_code = drain.reason_code if drain is not None else None + else: + settled = drain.entry_outcome(capture.entry_id) + if settled is None: + outcome = "pending" + reason_code = drain.reason_code + else: + outcome = str(settled["outcome"]) + reason_code = settled.get("reason_code") + return { + "schema_version": LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1, + "outcome": outcome, + "reason_code": reason_code, + "goal_id": goal_id, + "entry": { + "entry_id": capture.entry_id, + "partition": capture.partition, + "seq": capture.seq, + "partition_digest": capture.partition_digest, + "source_bytes_digest": capture.source_bytes_digest, + }, + "drain": None + if drain is None + else { + "outcome": drain.outcome, + "delivered": drain.delivered, + "replayed": drain.replayed, + "reclaimed_residue": drain.reclaimed_residue, + "pending_after": drain.pending_after, + "prepared_only_after": drain.prepared_only_after, + "stopped_at": drain.stopped_at, + "last_cursor": drain.last_cursor, + "provider_revision": drain.provider_revision, + "candidate_readback_verified": drain.candidate_readback_verified, + }, + "capture_kind": "source_transaction_outbox", + # Both flags are measured facts of this write: they are true only when + # a prepared/committed entry was actually recorded for it. A disabled + # or unchanged capture recorded nothing and claims nothing. + "source_transaction_correlated": capture.recorded, + "durable_source_outbox": capture.recorded, + "source_candidate_compared": False, + "parity_verdict": "not_evaluated", + "primary_authority": "legacy_local", + "candidate_provider": "file", + "candidate_read_for_decision": False, + "provider_to_local_writes": False, + "primary_writeback_preserved": True, + "store_identity": drain.store_identity if drain is not None else None, + } + + +def valid_evidence_v1(result: object, *, goal_id: str) -> bool: + """Closed-shape check for evidence v1 as attached to writer payloads.""" + + if not isinstance(result, dict): + return False + entry = result.get("entry") + return ( + result.get("schema_version") == LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1 + and result.get("outcome") in _EVIDENCE_V1_OUTCOMES + and result.get("goal_id") == goal_id + and isinstance(entry, dict) + and result.get("capture_kind") == "source_transaction_outbox" + and isinstance(result.get("source_transaction_correlated"), bool) + and isinstance(result.get("durable_source_outbox"), bool) + and result.get("source_candidate_compared") is False + and result.get("parity_verdict") == "not_evaluated" + and result.get("primary_authority") == "legacy_local" + and result.get("candidate_provider") == "file" + and result.get("candidate_read_for_decision") is False + and result.get("provider_to_local_writes") is False + and result.get("primary_writeback_preserved") is True + and (result.get("reason_code") is None or isinstance(result.get("reason_code"), str)) + ) + + +__all__ += [ + "CLI_DRAIN_LOCK_TIMEOUT_SECONDS", + "INLINE_DRAIN_BUDGET_SECONDS", + "INLINE_DRAIN_LOCK_TIMEOUT_SECONDS", + "INLINE_DRAIN_MAX_ENTRIES", + "LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_REQUEST_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_EVIDENCE_SCHEMA_V1", + "LOCAL_AUTHORITY_SHADOW_READ_REQUEST_SCHEMA", + "LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA", + "RETENTION_PRESSURE_BYTES", + "DrainResult", + "capture_evidence", + "primary_lock_is_free", + "todo_partition_projector", + "drain_local_authority_shadow_outbox", + "local_authority_shadow_status", + "read_local_authority_shadow", + "valid_evidence_v1", +] diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.py b/loopx/control_plane/coordination/local_authority_shadow_outbox.py new file mode 100644 index 0000000000..2aa84c721d --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.py @@ -0,0 +1,942 @@ +"""Durable, transaction-bound outbox for the local authority shadow. + +A legacy writer that changes coordination facts records, inside the lock it +already holds, a two-phase entry for exactly the state that lock guards (one +*partition*): a ``prepared`` file computed from the bytes about to be written, +then a ``committed`` marker after the primary write returns. Nothing in the +lock talks to the TypeScript runtime. A later drain (same process after the +lock, or an operator command) turns each committed entry into exactly one +candidate-store transaction whose ``operation_id`` is the entry id. + +The outbox never changes the primary verdict: every failure here is swallowed +into typed capture evidence and the primary write proceeds unchanged. +""" + +from __future__ import annotations + +import json +import os +import re +import uuid +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from .local_authority_shadow_projection import ( + LEASE_PARTITION, + PARTITIONS, + TODO_PARTITION, + ProjectionValueError, + canonical_value, + lease_partition_projection, + partition_digest, + sha256_digest, + text_digest, +) + + +OUTBOX_ENTRY_SCHEMA = "loopx_local_authority_shadow_outbox_entry_v0" +OUTBOX_COMMIT_SCHEMA = "loopx_local_authority_shadow_outbox_commit_v0" +DRAIN_CURSOR_SCHEMA = "loopx_local_authority_shadow_drain_cursor_v0" +SOURCE_MARKDOWN = "markdown_active_state" +SOURCE_STATE_EVENT_LOG = "state_event_log" +SOURCE_TASK_LEASE = "task_lease_record" +WRITER_RUNTIME_PYTHON = "python" +WRITER_RUNTIME_TYPESCRIPT = "typescript" +ENTRY_ID_PREFIX = "local-shadow-tx-" +_ENTRY_FILE = re.compile( + r"^(?P\d{10})-(?Plocal-shadow-tx-[0-9a-f]{64})\.(?Pprepared|committed)\.json$" +) +_LEASE_FILE = re.compile(r"^[A-Za-z0-9_.-]+\.json$") + + +class OutboxError(RuntimeError): + """Typed outbox failure; never escapes into a primary write.""" + + def __init__(self, reason_code: str, message: str) -> None: + super().__init__(message) + self.reason_code = reason_code + + +TodoPartitionProjector = Callable[[str], dict[str, Any]] +"""Active-state Markdown text -> ``{"handoff_mode": str, "todos": [compact...]}``.""" + + +def utc_now_text() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def outbox_root(runtime_root: Path, goal_id: str) -> Path: + return runtime_root / "authority-shadow" / "outbox" / goal_id + + +def partition_directory(runtime_root: Path, goal_id: str, partition: str) -> Path: + if partition not in PARTITIONS: + raise OutboxError("invalid_partition", f"unknown outbox partition {partition!r}") + return outbox_root(runtime_root, goal_id) / partition + + +def drain_lock_target(runtime_root: Path, goal_id: str) -> Path: + return outbox_root(runtime_root, goal_id) / "drain" + + +def lease_directory(runtime_root: Path, goal_id: str) -> Path: + return runtime_root / "goals" / goal_id / "task-leases" + + +def entry_identity(*, goal_id: str, partition: str, seq: int, source_ref: str) -> str: + """Bind the entry id to the exact primary bytes (or event) it records.""" + + return ENTRY_ID_PREFIX + sha256_digest( + {"goal_id": goal_id, "partition": partition, "seq": seq, "source_ref": source_ref} + ).removeprefix("sha256:") + + +def entry_file_name(seq: int, entry_id: str, phase: str) -> str: + return f"{seq:010d}-{entry_id}.{phase}.json" + + +def _fsync_directory(directory: Path) -> None: + if os.name == "nt": + return + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def durable_write_json(path: Path, payload: Mapping[str, Any]) -> None: + """Temp file in the same directory, fsync, atomic replace, directory fsync.""" + + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = path.with_name(f"{path.name}.tmp-{os.getpid()}-{uuid.uuid4().hex}") + data = json.dumps(payload, ensure_ascii=False, sort_keys=True, indent=1).encode("utf-8") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + view = memoryview(data) + while view: + written = os.write(descriptor, view) + view = view[written:] + os.fsync(descriptor) + finally: + os.close(descriptor) + try: + os.replace(temporary, path) + _fsync_directory(path.parent) + finally: + if temporary.exists(): + temporary.unlink(missing_ok=True) + + +def _as_object(value: object) -> dict[str, Any]: + """Narrow an untyped JSON value to an object; anything else is empty.""" + + return dict(value) if isinstance(value, Mapping) else {} + + +def _load_json(path: Path) -> dict[str, Any]: + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise OutboxError("outbox_file_invalid", f"{path.name} is not a JSON object") + return raw + + +@dataclass(frozen=True, slots=True) +class OutboxEntry: + """One two-phase entry as found on disk.""" + + partition: str + seq: int + entry_id: str + prepared_path: Path + committed_path: Path | None + prepared: dict[str, Any] + committed: dict[str, Any] | None + + @property + def is_committed(self) -> bool: + return self.committed is not None + + @property + def source_ref(self) -> str: + return str(record_source_ref(self.prepared)) + + def projection(self) -> dict[str, Any] | None: + """The partition projection recorded for this entry, if any.""" + + for record in (self.committed, self.prepared): + if isinstance(record, dict) and isinstance(record.get("projection"), dict): + return dict(record["projection"]) + return None + + def recorded_partition_digest(self) -> str | None: + for record in (self.committed, self.prepared): + if isinstance(record, dict) and isinstance(record.get("partition_digest"), str): + return str(record["partition_digest"]) + return None + + +_EntryKey = tuple[int, str] + + +def _index_entry_files(directory: Path) -> tuple[dict[_EntryKey, Path], dict[_EntryKey, Path]]: + """Map ``(seq, entry_id)`` to the prepared and committed files present.""" + + prepared: dict[_EntryKey, Path] = {} + committed: dict[_EntryKey, Path] = {} + for path in directory.iterdir(): + match = _ENTRY_FILE.match(path.name) + if match is None: + continue + key = (int(match.group("seq")), match.group("entry_id")) + target = prepared if match.group("phase") == "prepared" else committed + target[key] = path + return prepared, committed + + +_WRITER_RUNTIMES = frozenset({WRITER_RUNTIME_PYTHON, WRITER_RUNTIME_TYPESCRIPT}) +_SOURCE_KINDS = frozenset({SOURCE_MARKDOWN, SOURCE_STATE_EVENT_LOG, SOURCE_TASK_LEASE}) +_DIGEST_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") + + +def _load_prepared_record( + path: Path, + *, + directory: Path, + seq: int, + entry_id: str, +) -> dict[str, Any]: + """Load one prepared record and prove it binds this directory and entry id. + + The file name, the directory (``outbox//``), the record's + own goal/partition fields, and the entry id recomputed from the record's + source reference must all agree; a record that was copied, edited, or + written for another goal fails closed before it can reach the candidate. + """ + + record = _load_json(path) + writer = _as_object(record.get("writer")) + source = _as_object(record.get("source")) + expected_goal = directory.parent.name + expected_partition = directory.name + source_ref = record_source_ref(record) + root_digest = record.get("source_root_digest") + bound = ( + record.get("schema_version") == OUTBOX_ENTRY_SCHEMA + and record.get("entry_id") == entry_id + and record.get("seq") == seq + and record.get("goal_id") == expected_goal + and record.get("partition") == expected_partition + and writer.get("runtime") in _WRITER_RUNTIMES + and isinstance(writer.get("write_class"), str) + and bool(writer.get("write_class")) + and source.get("kind") in _SOURCE_KINDS + and isinstance(root_digest, str) + and _DIGEST_PATTERN.match(root_digest) is not None + and source_ref is not None + and entry_identity( + goal_id=expected_goal, + partition=expected_partition, + seq=seq, + source_ref=source_ref, + ) + == entry_id + ) + if not bound: + raise OutboxError( + "outbox_file_invalid", + f"{path.name} does not bind its directory, identity, and source reference", + ) + return record + + +def _load_committed_record(path: Path | None, *, entry_id: str) -> dict[str, Any] | None: + if path is None: + return None + record = _load_json(path) + if record.get("schema_version") != OUTBOX_COMMIT_SCHEMA or record.get("entry_id") != entry_id: + raise OutboxError("outbox_file_invalid", f"{path.name} does not match its entry") + return record + + +def _retired_watermark(directory: Path) -> int: + cursor = read_cursor(directory) + return int(cursor.get("last_seq") or 0) if cursor is not None else 0 + + +def retired_residue(directory: Path) -> list[Path]: + """Entry files at or below the durable cursor. + + The cursor is written before an entry's files are unlinked, so anything it + covers is already settled in the candidate store. A crash between the + cursor write and the unlinks, or between the two unlinks, leaves these + files behind; they are residue to reclaim, never entries to deliver or + markers to reject. + """ + + if not directory.is_dir(): + return [] + watermark = _retired_watermark(directory) + prepared, committed = _index_entry_files(directory) + residue = [ + path + for key, path in [*prepared.items(), *committed.items()] + if key[0] <= watermark + ] + return sorted(residue) + + +def reclaim_retired_residue(directory: Path) -> int: + """Unlink retired residue; the caller must hold the goal's drain lock.""" + + residue = retired_residue(directory) + for path in residue: + path.unlink(missing_ok=True) + return len(residue) + + +def list_entries(directory: Path) -> list[OutboxEntry]: + """All live entries of one partition directory, oldest first. + + Files the durable cursor already covers are retired residue and are not + listed; a committed marker without a prepared entry above the cursor is + real corruption and fails closed. + """ + + if not directory.is_dir(): + return [] + watermark = _retired_watermark(directory) + prepared, committed = _index_entry_files(directory) + prepared = {key: path for key, path in prepared.items() if key[0] > watermark} + committed = {key: path for key, path in committed.items() if key[0] > watermark} + orphan_markers = sorted(set(committed) - set(prepared)) + if orphan_markers: + seq, entry_id = orphan_markers[0] + raise OutboxError( + "outbox_file_invalid", + f"committed marker without prepared entry: {entry_file_name(seq, entry_id, 'committed')}", + ) + entries: list[OutboxEntry] = [] + for seq, entry_id in sorted(prepared): + key = (seq, entry_id) + prepared_record = _load_prepared_record( + prepared[key], directory=directory, seq=seq, entry_id=entry_id + ) + entries.append( + OutboxEntry( + partition=str(prepared_record.get("partition") or directory.name), + seq=seq, + entry_id=entry_id, + prepared_path=prepared[key], + committed_path=committed.get(key), + prepared=prepared_record, + committed=_load_committed_record(committed.get(key), entry_id=entry_id), + ) + ) + return entries + + +def cursor_path(directory: Path) -> Path: + return directory / "drain-cursor.json" + + +def read_cursor(directory: Path) -> dict[str, Any] | None: + path = cursor_path(directory) + if not path.exists(): + return None + record = _load_json(path) + if record.get("schema_version") != DRAIN_CURSOR_SCHEMA: + raise OutboxError("outbox_file_invalid", "drain cursor schema is unsupported") + return record + + +def write_cursor( + directory: Path, + *, + partition: str, + last_seq: int, + last_entry_id: str, + last_partition_digest: str | None, + last_cursor: str | None, + last_provider_revision: str | None, +) -> None: + durable_write_json( + cursor_path(directory), + { + "schema_version": DRAIN_CURSOR_SCHEMA, + "partition": partition, + "last_seq": last_seq, + "last_entry_id": last_entry_id, + "last_partition_digest": last_partition_digest, + "last_cursor": last_cursor, + "last_provider_revision": last_provider_revision, + "updated_at": utc_now_text(), + }, + ) + + +def next_seq(directory: Path) -> int: + """Gap-free sequence: past the newest file and the drained watermark.""" + + highest = 0 + if directory.is_dir(): + for path in directory.iterdir(): + match = _ENTRY_FILE.match(path.name) + if match is not None: + highest = max(highest, int(match.group("seq"))) + cursor = read_cursor(directory) + if cursor is not None: + highest = max(highest, int(cursor.get("last_seq") or 0)) + return highest + 1 + + +def latest_partition_digest(directory: Path) -> str | None: + """Digest of the newest known partition state (pending entry, else cursor).""" + + entries = list_entries(directory) + for entry in reversed(entries): + digest = entry.recorded_partition_digest() + if digest is not None: + return digest + cursor = read_cursor(directory) + if cursor is not None and isinstance(cursor.get("last_partition_digest"), str): + return str(cursor["last_partition_digest"]) + return None + + +def runtime_root_digest(runtime_root: Path) -> str: + """Digest of the absolute, dot-normalized root; must match the TypeScript writer. + + Symlinks are deliberately not resolved: both runtimes normalize the string + they were given, so a root passed through the effect runtime hashes the + same on either side. + """ + + return text_digest(os.path.abspath(str(runtime_root))) + + +def record_source_ref(record: Mapping[str, Any]) -> str | None: + """The source reference an entry id binds: bytes digest, event id, or seed digest.""" + + source = _as_object(record.get("source")) + bytes_digest = source.get("bytes_digest") + if isinstance(bytes_digest, str) and bytes_digest: + return bytes_digest + event_id = source.get("event_id") + if isinstance(event_id, str) and event_id: + return f"event:{event_id}" + digest = record.get("partition_digest") + if isinstance(digest, str) and digest: + return f"seed:{digest}" + return None + + +def read_lease_records(directory: Path) -> list[tuple[str, dict[str, Any]]]: + """Top-level lease records of a goal; lifecycle receipts are excluded.""" + + if not directory.is_dir(): + return [] + records: list[tuple[str, dict[str, Any]]] = [] + for path in sorted(directory.iterdir()): + if not path.is_file() or not _LEASE_FILE.match(path.name) or path.name.startswith("."): + continue + raw = json.loads(path.read_text(encoding="utf-8")) + if isinstance(raw, dict): + records.append((path.stem, raw)) + return records + + +def compact_lease_projection( + raw_projection: Mapping[str, Any], *, goal_id: str +) -> dict[str, Any]: + """Compact the TypeScript-written lease partition (``{leases: [{file_stem, record}]}``).""" + + raw_leases = raw_projection.get("leases") + if not isinstance(raw_leases, list): + raise OutboxError("outbox_file_invalid", "lease partition projection must list leases") + records: list[tuple[str, object]] = [] + for item in raw_leases: + if not isinstance(item, dict): + raise OutboxError("outbox_file_invalid", "lease projection item must be an object") + stem = item.get("file_stem") + if not isinstance(stem, str) or not stem: + raise OutboxError("outbox_file_invalid", "lease projection item needs a file_stem") + records.append((stem, item.get("record"))) + try: + return lease_partition_projection(records, goal_id=goal_id) + except ProjectionValueError as error: + raise OutboxError("outbox_file_invalid", str(error)) from error + + +def _writer(write_class: str, *, runtime: str, operation_id: str | None) -> dict[str, Any]: + return {"runtime": runtime, "write_class": write_class, "operation_id": operation_id} + + +def _entry_record( + *, + goal_id: str, + partition: str, + seq: int, + entry_id: str, + writer: Mapping[str, Any], + source: Mapping[str, Any], + source_root_digest: str, + projection: Mapping[str, Any] | None, + digest: str | None, +) -> dict[str, Any]: + return { + "schema_version": OUTBOX_ENTRY_SCHEMA, + "goal_id": goal_id, + "partition": partition, + "seq": seq, + "entry_id": entry_id, + "writer": dict(writer), + "source": dict(source), + "source_root_digest": source_root_digest, + "projection": canonical_value(dict(projection)) if projection is not None else None, + "partition_digest": digest, + "prepared_at": utc_now_text(), + } + + +@dataclass +class CaptureOutcome: + """What the capture did for one primary write (attached to its evidence).""" + + entry_id: str | None = None + partition: str | None = None + seq: int | None = None + partition_digest: str | None = None + source_bytes_digest: str | None = None + skipped_reason: str | None = None + failure: dict[str, Any] | None = None + + @property + def recorded(self) -> bool: + return self.entry_id is not None and self.failure is None + + +class TodoPartitionCapture: + """Two-phase capture of the todos partition inside the active-state lock. + + ``begin`` returns an inert capture when the goal has no shadow binding, so + default-off writers create no directory, lock, or file. + """ + + def __init__( + self, + *, + enabled: bool, + runtime_root: Path | None, + goal_id: str, + state_path: Path | None, + write_class: str, + original_text: str, + projector: TodoPartitionProjector | None, + ) -> None: + self._enabled = enabled + self._runtime_root = runtime_root + self._goal_id = goal_id + self._state_path = state_path + self._write_class = write_class + self._original_digest = text_digest(original_text) + self._projector = projector + self._directory = ( + partition_directory(runtime_root, goal_id, TODO_PARTITION) + if enabled and runtime_root is not None + else None + ) + self._seq: int | None = None + self._entry_id: str | None = None + self._event_id: str | None = None + self.outcome = CaptureOutcome(partition=TODO_PARTITION if enabled else None) + + @classmethod + def begin( + cls, + *, + enabled: bool, + runtime_root: Path | None, + goal_id: str, + state_path: Path | None, + write_class: str, + original_text: str, + projector: TodoPartitionProjector | None, + ) -> TodoPartitionCapture: + """``projector`` maps active-state text to the todos partition projection. + + It is injected so this module stays free of the Markdown parser; the + adapter supplies the production projector. + """ + + return cls( + enabled=enabled, + runtime_root=runtime_root, + goal_id=goal_id, + state_path=state_path, + write_class=write_class, + original_text=original_text, + projector=projector, + ) + + @property + def enabled(self) -> bool: + return self._enabled and self._directory is not None + + def skip(self, reason: str) -> None: + """Record why this writer deliberately did not open a transaction.""" + + if self.enabled and self.outcome.entry_id is None: + self.outcome.skipped_reason = reason + + def _project(self, state_text: str) -> dict[str, Any]: + if self._projector is None: + raise OutboxError("outbox_prepare_failed", "a todo partition projector is required") + projection = self._projector(state_text) + if set(projection) != {"handoff_mode", "todos"}: + raise OutboxError("outbox_prepare_failed", "projector must return {handoff_mode, todos}") + return projection + + def _fail(self, reason_code: str, error: BaseException) -> None: + self.outcome.failure = { + "reason_code": reason_code, + "error_class": error.__class__.__name__, + } + + def prepare(self, new_text: str, *, event_id: str | None = None) -> None: + """Record the prepared entry for the bytes about to be written. + + For the state-event-log branch pass ``new_text=original`` plus the + event id; the projection is then recorded by ``committed`` after the + append, still inside the same lock. + """ + + if not self.enabled or self._directory is None or self._runtime_root is None: + self.outcome.skipped_reason = "shadow_disabled" + return + try: + if event_id is None: + projection = self._project(new_text) + digest = partition_digest(projection) + if digest == latest_partition_digest(self._directory): + self.outcome.skipped_reason = "partition_unchanged" + return + source_ref = text_digest(new_text) + bytes_digest: str | None = source_ref + source_kind = SOURCE_MARKDOWN + else: + projection = None + digest = None + bytes_digest = None + source_kind = SOURCE_STATE_EVENT_LOG + source_ref = f"event:{event_id}" + seq = next_seq(self._directory) + entry_id = entry_identity( + goal_id=self._goal_id, + partition=TODO_PARTITION, + seq=seq, + source_ref=source_ref, + ) + record = _entry_record( + goal_id=self._goal_id, + partition=TODO_PARTITION, + seq=seq, + entry_id=entry_id, + writer=_writer( + self._write_class, + runtime=WRITER_RUNTIME_PYTHON, + operation_id=event_id, + ), + source={ + "kind": source_kind, + "previous_bytes_digest": self._original_digest, + "bytes_digest": bytes_digest, + "lease": None, + "event_id": event_id, + }, + source_root_digest=runtime_root_digest(self._runtime_root), + projection=projection, + digest=digest, + ) + durable_write_json( + self._directory / entry_file_name(seq, entry_id, "prepared"), + record, + ) + except Exception as error: # noqa: BLE001 - the primary write must proceed + self._fail("outbox_prepare_failed", error) + return + self._seq = seq + self._entry_id = entry_id + self._event_id = event_id + self.outcome.entry_id = entry_id + self.outcome.seq = seq + self.outcome.partition_digest = digest + self.outcome.source_bytes_digest = bytes_digest + + def committed(self, *, projection_from_disk: bool = False) -> None: + """Mark the prepared entry committed after the primary write returned.""" + + if self._seq is None or self._entry_id is None or self._directory is None: + return + if self.outcome.failure is not None: + return + marker: dict[str, Any] = { + "schema_version": OUTBOX_COMMIT_SCHEMA, + "entry_id": self._entry_id, + "committed_at": utc_now_text(), + } + try: + if projection_from_disk: + if self._state_path is None: + raise OutboxError("outbox_commit_marker_failed", "state path is required") + projection = self._project(self._state_path.read_text(encoding="utf-8")) + digest = partition_digest(projection) + if digest == latest_partition_digest(self._directory): + # The event changed nothing the shadow compares; retire the + # prepared entry so no crash-window resolution is needed. + (self._directory / entry_file_name(self._seq, self._entry_id, "prepared")).unlink( + missing_ok=True + ) + self.outcome.entry_id = None + self.outcome.seq = None + self.outcome.skipped_reason = "partition_unchanged" + return + marker["projection"] = canonical_value(projection) + marker["partition_digest"] = digest + self.outcome.partition_digest = digest + durable_write_json( + self._directory / entry_file_name(self._seq, self._entry_id, "committed"), + marker, + ) + except Exception as error: # noqa: BLE001 - the primary write already landed + self._fail("outbox_commit_marker_failed", error) + + +SourceProbe = Callable[[OutboxEntry], str] +"""Return ``committed``, ``abandoned`` or ``unproved`` for a prepared-only entry.""" + + +_LEASE_FENCE_KEYS = ("version", "lease_epoch", "status", "updated_at") + + +def _resolve_markdown_source(source: Mapping[str, Any], reader: Callable[[], str]) -> str: + current_digest = text_digest(reader()) + if current_digest == source.get("bytes_digest"): + return "committed" + if current_digest == source.get("previous_bytes_digest"): + return "abandoned" + return "unproved" + + +def _lease_matches(current: Mapping[str, Any] | None, expected: Mapping[str, Any]) -> bool: + if current is None or not expected: + return False + return all(current.get(key) == expected.get(key) for key in _LEASE_FENCE_KEYS) + + +def _resolve_lease_source( + source: Mapping[str, Any], + reader: Callable[[str], dict[str, Any] | None], +) -> str: + planned = _as_object(source.get("lease")) + if not planned: + return "unproved" + current = reader(str(planned.get("todo_id") or "")) + if _lease_matches(current, planned): + return "committed" + previous = _as_object(source.get("previous_lease")) + if (not previous and current is None) or _lease_matches(current, previous): + return "abandoned" + return "unproved" + + +def _resolve_event_source(source: Mapping[str, Any], reader: Callable[[str], bool]) -> str: + event_id = source.get("event_id") + if isinstance(event_id, str) and event_id and reader(event_id): + # The append landed but the projection was never recorded; only a + # fresh full-partition capture can say what the state now is. + return "unproved" + return "abandoned" + + +def resolve_prepared_only_entry( + entry: OutboxEntry, + *, + markdown_text_reader: Callable[[], str] | None, + lease_record_reader: Callable[[str], dict[str, Any] | None] | None, + event_presence_reader: Callable[[str], bool] | None, +) -> str: + """Decide what a prepared entry without a committed marker means. + + The caller must hold the partition's primary lock (or have proven it free), + otherwise the source may still be mid-write. + """ + + source = _as_object(entry.prepared.get("source")) + kind = source.get("kind") + if kind == SOURCE_MARKDOWN and markdown_text_reader is not None: + return _resolve_markdown_source(source, markdown_text_reader) + if kind == SOURCE_TASK_LEASE and lease_record_reader is not None: + return _resolve_lease_source(source, lease_record_reader) + if kind == SOURCE_STATE_EVENT_LOG and event_presence_reader is not None: + return _resolve_event_source(source, event_presence_reader) + return "unproved" + + +def entries_by_partition(runtime_root: Path, goal_id: str) -> dict[str, list[OutboxEntry]]: + return { + partition: list_entries(partition_directory(runtime_root, goal_id, partition)) + for partition in PARTITIONS + } + + +def outbox_summary(runtime_root: Path, goal_id: str) -> dict[str, Any]: + """Counts per partition for operator readback; never raises on an empty outbox.""" + + summary: dict[str, Any] = {} + for partition in PARTITIONS: + directory = partition_directory(runtime_root, goal_id, partition) + try: + entries = list_entries(directory) + cursor = read_cursor(directory) + invalid: str | None = None + except OutboxError as error: + entries, cursor, invalid = [], None, error.reason_code + summary[partition] = { + "committed_pending": sum(1 for entry in entries if entry.is_committed), + "prepared_only": sum(1 for entry in entries if not entry.is_committed), + "retired_residue": len(retired_residue(directory)), + "next_seq": (max((entry.seq for entry in entries), default=0) if entries else 0), + "cursor_last_seq": int(cursor.get("last_seq") or 0) if cursor else None, + "cursor_last_entry_id": cursor.get("last_entry_id") if cursor else None, + "invalid": invalid, + } + return summary + + +def remove_entry_files(entry: OutboxEntry) -> None: + entry.prepared_path.unlink(missing_ok=True) + if entry.committed_path is not None: + entry.committed_path.unlink(missing_ok=True) + + +@dataclass(frozen=True, slots=True) +class SeedSource: + """A full-partition snapshot taken under the partition's primary lock.""" + + partition: str + projection: dict[str, Any] + source_bytes_digest: str | None = None + extra_source: dict[str, Any] = field(default_factory=dict) + + +def write_seed_entry( + *, + runtime_root: Path, + goal_id: str, + seed: SeedSource, + write_class: str = "seed", +) -> OutboxEntry: + """Write a committed full-partition entry (seed or reseed); caller holds the lock.""" + + directory = partition_directory(runtime_root, goal_id, seed.partition) + digest = partition_digest(seed.projection) + seq = next_seq(directory) + source_ref: str = seed.source_bytes_digest if seed.source_bytes_digest else f"seed:{digest}" + entry_id = entry_identity(goal_id=goal_id, partition=seed.partition, seq=seq, source_ref=source_ref) + record = _entry_record( + goal_id=goal_id, + partition=seed.partition, + seq=seq, + entry_id=entry_id, + writer=_writer(write_class, runtime=WRITER_RUNTIME_PYTHON, operation_id=None), + source={ + "kind": SOURCE_MARKDOWN if seed.partition == TODO_PARTITION else SOURCE_TASK_LEASE, + "previous_bytes_digest": None, + "bytes_digest": seed.source_bytes_digest, + "lease": None, + "event_id": None, + **dict(seed.extra_source), + }, + source_root_digest=runtime_root_digest(runtime_root), + projection=seed.projection, + digest=digest, + ) + prepared_path = directory / entry_file_name(seq, entry_id, "prepared") + committed_path = directory / entry_file_name(seq, entry_id, "committed") + durable_write_json(prepared_path, record) + marker = { + "schema_version": OUTBOX_COMMIT_SCHEMA, + "entry_id": entry_id, + "committed_at": utc_now_text(), + } + durable_write_json(committed_path, marker) + return OutboxEntry( + partition=seed.partition, + seq=seq, + entry_id=entry_id, + prepared_path=prepared_path, + committed_path=committed_path, + prepared=record, + committed=marker, + ) + + +def lease_seed_source(runtime_root: Path, goal_id: str) -> SeedSource: + """Snapshot the lease partition from disk; caller holds the lease lock.""" + + records = read_lease_records(lease_directory(runtime_root, goal_id)) + try: + projection = lease_partition_projection(records, goal_id=goal_id) + except ProjectionValueError as error: + raise OutboxError("outbox_prepare_failed", str(error)) from error + return SeedSource(partition=LEASE_PARTITION, projection=projection) + + +def iter_committed(entries: Iterable[OutboxEntry]) -> list[OutboxEntry]: + return [entry for entry in entries if entry.is_committed] + + +__all__ = [ + "DRAIN_CURSOR_SCHEMA", + "OUTBOX_COMMIT_SCHEMA", + "OUTBOX_ENTRY_SCHEMA", + "SOURCE_MARKDOWN", + "SOURCE_STATE_EVENT_LOG", + "SOURCE_TASK_LEASE", + "CaptureOutcome", + "OutboxEntry", + "OutboxError", + "SeedSource", + "TodoPartitionCapture", + "TodoPartitionProjector", + "compact_lease_projection", + "drain_lock_target", + "durable_write_json", + "entries_by_partition", + "entry_file_name", + "entry_identity", + "iter_committed", + "latest_partition_digest", + "lease_directory", + "lease_seed_source", + "list_entries", + "next_seq", + "outbox_root", + "outbox_summary", + "partition_directory", + "read_cursor", + "read_lease_records", + "reclaim_retired_residue", + "record_source_ref", + "remove_entry_files", + "resolve_prepared_only_entry", + "retired_residue", + "runtime_root_digest", + "utc_now_text", + "write_cursor", + "write_seed_entry", +] diff --git a/loopx/control_plane/coordination/local_authority_shadow_outbox.ts b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts new file mode 100644 index 0000000000..5688377508 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_outbox.ts @@ -0,0 +1,272 @@ +import { createHash } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import type { JsonObject } from "../effect_program.ts"; +import { durableWriteJson } from "../effect_runtime_io.ts"; +import { authorityUnicodeCompare, canonicalAuthorityBytes } from "./authority_store_codec.ts"; + +/** + * Lease-partition side of the local authority shadow outbox. + * + * The task-lease writers already hold the goal's lease lock when they persist + * a record; this module lets them append a two-phase outbox entry for exactly + * that partition inside the same lock. It never touches the candidate store, + * never blocks, and never throws into the lease write: every failure is + * returned on the capture object so the writer can attach typed evidence. + */ + +export const LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA = + "loopx_coordination_runtime_shadow_binding_v0"; +export const LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA = + "loopx_local_authority_shadow_outbox_entry_v0"; +export const LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA = + "loopx_local_authority_shadow_outbox_commit_v0"; +export const LOCAL_AUTHORITY_SHADOW_DRAIN_CURSOR_SCHEMA = + "loopx_local_authority_shadow_drain_cursor_v0"; +export const LEASE_PARTITION = "leases"; +const ENTRY_FILE = /^(\d{10})-(local-shadow-tx-[0-9a-f]{64})\.(prepared|committed)\.json$/u; +const LEASE_FILE = /^[A-Za-z0-9_.-]+\.json$/u; +const LEASE_SOURCE_FIELDS = ["todo_id", "version", "lease_epoch", "status", "updated_at"] as const; + +export interface LocalAuthorityShadowBinding { + schema_version: typeof LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA; + provider: "file_v0"; +} + +/** Decode the optional per-request binding; anything but the exact contract is "absent". */ +export function decodeLocalAuthorityShadowBinding( + value: unknown, +): LocalAuthorityShadowBinding | null { + if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const record = value as Record; + const keys = Object.keys(record); + if ( + keys.length !== 2 || + record.schema_version !== LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA || + record.provider !== "file_v0" + ) { + return null; + } + return { schema_version: LOCAL_AUTHORITY_SHADOW_BINDING_SCHEMA, provider: "file_v0" }; +} + +export function sha256Digest(input: Uint8Array | string): string { + return `sha256:${createHash("sha256").update(input).digest("hex")}`; +} + +/** Digest of the exact bytes `atomicWriteJson` persists for a lease record. */ +export function leaseRecordDigest(record: JsonObject): string { + return sha256Digest(`${JSON.stringify(record, null, 2)}\n`); +} + +/** Must stay byte-compatible with the Python `entry_identity` derivation. */ +export function outboxEntryIdentity( + goalId: string, + partition: string, + seq: number, + sourceRef: string, +): string { + const digest = createHash("sha256") + .update(canonicalAuthorityBytes({ + goal_id: goalId, + partition, + seq, + source_ref: sourceRef, + })) + .digest("hex"); + return `local-shadow-tx-${digest}`; +} + +export function outboxPartitionDirectory( + runtimeRoot: string, + goalId: string, + partition: string, +): string { + return join(runtimeRoot, "authority-shadow", "outbox", goalId, partition); +} + +export function outboxEntryFileName(seq: number, entryId: string, phase: "prepared" | "committed"): string { + return `${String(seq).padStart(10, "0")}-${entryId}.${phase}.json`; +} + +function isMissing(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === "ENOENT"; +} + +async function nextSeq(directory: string): Promise { + let highest = 0; + try { + for (const name of await readdir(directory)) { + const match = ENTRY_FILE.exec(name); + if (match !== null) highest = Math.max(highest, Number(match[1])); + } + } catch (error) { + if (!isMissing(error)) throw error; + } + try { + const raw: unknown = JSON.parse(await readFile(join(directory, "drain-cursor.json"), "utf8")); + if (raw !== null && typeof raw === "object") { + const lastSeq = (raw as Record).last_seq; + if (typeof lastSeq === "number" && Number.isSafeInteger(lastSeq)) { + highest = Math.max(highest, lastSeq); + } + } + } catch (error) { + if (!isMissing(error)) throw error; + } + return highest + 1; +} + +async function readLeasePartition( + leaseDirectory: string, + plannedStem: string, + plannedLease: JsonObject, +): Promise { + const records = new Map(); + let names: string[] = []; + try { + names = await readdir(leaseDirectory); + } catch (error) { + if (!isMissing(error)) throw error; + } + for (const name of names) { + if (!LEASE_FILE.test(name) || name.startsWith(".")) continue; + const stem = name.slice(0, -".json".length); + if (stem === plannedStem) continue; + const raw: unknown = JSON.parse(await readFile(join(leaseDirectory, name), "utf8")); + if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) { + records.set(stem, raw as JsonObject); + } + } + records.set(plannedStem, plannedLease); + const stems = [...records.keys()]; + stems.sort(authorityUnicodeCompare); + return stems.map((stem) => ({ file_stem: stem, record: records.get(stem) as JsonObject })); +} + +function leaseSourceFacts(record: JsonObject | null): JsonObject | null { + if (record === null) return null; + const facts: JsonObject = {}; + for (const field of LEASE_SOURCE_FIELDS) { + const value = record[field]; + if (value === undefined) continue; + if (typeof value === "string" || typeof value === "number" || value === null) { + facts[field] = value; + } + } + return facts; +} + +export interface LeaseOutboxCaptureInput { + runtime_root: string; + goal_id: string; + lease_directory: string; + write_class: string; + operation_id: string | null; + previous_lease: JsonObject | null; + planned_lease: JsonObject; +} + +export interface LeaseOutboxCapture { + entry_id: string | null; + seq: number | null; + source_bytes_digest: string | null; + failure: { reason_code: string; error_class: string } | null; + /** Write the committed marker after the lease record landed. Never throws. */ + commit(): Promise; +} + +function failureOf(reasonCode: string, error: unknown): { reason_code: string; error_class: string } { + return { + reason_code: reasonCode, + error_class: error instanceof Error ? error.constructor.name : typeof error, + }; +} + +/** + * Record a prepared lease-partition entry for the record about to be written. + * + * Call inside the lease lock, right before `atomicWriteJson(leasePath, planned)`; + * call `commit()` right after it returns. A returned failure never blocks the + * primary write. + */ +export async function beginLeaseOutboxEntry( + input: LeaseOutboxCaptureInput, +): Promise { + const inert: LeaseOutboxCapture = { + entry_id: null, + seq: null, + source_bytes_digest: null, + failure: null, + async commit(): Promise {}, + }; + const plannedStem = typeof input.planned_lease.todo_id === "string" + ? input.planned_lease.todo_id + : ""; + if (plannedStem.length === 0) { + return { ...inert, failure: { reason_code: "outbox_prepare_failed", error_class: "MissingTodoId" } }; + } + const directory = outboxPartitionDirectory(input.runtime_root, input.goal_id, LEASE_PARTITION); + try { + const projection = { leases: await readLeasePartition(input.lease_directory, plannedStem, input.planned_lease) }; + const bytesDigest = leaseRecordDigest(input.planned_lease); + const seq = await nextSeq(directory); + const entryId = outboxEntryIdentity(input.goal_id, LEASE_PARTITION, seq, bytesDigest); + const entry: JsonObject = { + schema_version: LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA, + goal_id: input.goal_id, + partition: LEASE_PARTITION, + seq, + entry_id: entryId, + writer: { + runtime: "typescript", + write_class: input.write_class, + operation_id: input.operation_id, + }, + source: { + kind: "task_lease_record", + previous_bytes_digest: input.previous_lease === null + ? null + : leaseRecordDigest(input.previous_lease), + bytes_digest: bytesDigest, + lease: leaseSourceFacts(input.planned_lease), + previous_lease: leaseSourceFacts(input.previous_lease), + event_id: null, + }, + source_root_digest: sha256Digest(resolve(input.runtime_root)), + projection, + partition_digest: null, + prepared_at: new Date().toISOString(), + }; + await durableWriteJson(join(directory, outboxEntryFileName(seq, entryId, "prepared")), entry); + let committed = false; + const capture: LeaseOutboxCapture = { + entry_id: entryId, + seq, + source_bytes_digest: bytesDigest, + failure: null, + async commit(): Promise { + if (committed) return; + committed = true; + try { + await durableWriteJson( + join(directory, outboxEntryFileName(seq, entryId, "committed")), + { + schema_version: LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA, + entry_id: entryId, + committed_at: new Date().toISOString(), + }, + ); + } catch (error) { + capture.failure = failureOf("outbox_commit_marker_failed", error); + } + }, + }; + return capture; + } catch (error) { + return { ...inert, failure: failureOf("outbox_prepare_failed", error) }; + } +} diff --git a/loopx/control_plane/coordination/local_authority_shadow_projection.py b/loopx/control_plane/coordination/local_authority_shadow_projection.py new file mode 100644 index 0000000000..0763f08a86 --- /dev/null +++ b/loopx/control_plane/coordination/local_authority_shadow_projection.py @@ -0,0 +1,171 @@ +"""Pure projection rules shared by the local authority shadow capture and parity. + +Everything here is a deterministic function of its inputs: no file, lock, +registry, or effect-runtime access. The same complete record contracts and canonical +bytes define the source digest, the outbox partition digest, and the candidate +readback comparison, so no two code paths can disagree about what "the same +coordination state" means. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Mapping +from typing import Any + +from ..todos.todo_summary import TODO_CANONICAL_READ_RECORD_FIELDS, canonical_todo_read_record + + +LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V0 = "loopx_local_authority_shadow_projection_v0" +TODO_PARTITION = "todos" +LEASE_PARTITION = "leases" +PARTITIONS: tuple[str, ...] = (TODO_PARTITION, LEASE_PARTITION) + +TODO_FIELDS: tuple[str, ...] = TODO_CANONICAL_READ_RECORD_FIELDS + + +class ProjectionValueError(ValueError): + """A value cannot be part of a canonical shadow projection.""" + + +def _reject_floats(value: object, path: str) -> None: + # Python `1.0` and JavaScript `1` would canonicalize differently, so a + # float anywhere in a compared projection would manufacture a false + # divergence between the Python source digest and the TypeScript head. + if isinstance(value, bool) or value is None or isinstance(value, (str, int)): + return + if isinstance(value, float): + raise ProjectionValueError(f"float values are not allowed in shadow projections ({path})") + if isinstance(value, Mapping): + for key, item in value.items(): + if not isinstance(key, str): + raise ProjectionValueError(f"non-string key in shadow projection ({path})") + _reject_floats(item, f"{path}.{key}") + return + if isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _reject_floats(item, f"{path}[{index}]") + return + raise ProjectionValueError( + f"unsupported value type {type(value).__name__} in shadow projection ({path})" + ) + + +def canonical_bytes(value: object) -> bytes: + """Sorted-key, minimal-separator UTF-8 JSON; floats and NaN are rejected.""" + + _reject_floats(value, "$") + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def canonical_value(value: object) -> Any: + """Round-trip a value through canonical JSON so key order is normalized.""" + + return json.loads(canonical_bytes(value)) + + +def sha256_digest(value: object) -> str: + return "sha256:" + hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def text_digest(text: str) -> str: + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def compact_todo(raw: object) -> dict[str, Any] | None: + """Retain the complete versioned Todo consumer record.""" + + if not isinstance(raw, Mapping): + return None + todo_id = str(raw.get("todo_id") or "").strip() + if not todo_id: + return None + try: + return dict(canonical_value(canonical_todo_read_record(dict(raw)))) + except ValueError as error: + raise ProjectionValueError(str(error)) from error + + +def compact_lease(raw: object, *, goal_id: str, file_stem: str) -> dict[str, Any]: + """Retain the complete versioned lease record after binding its identity.""" + + if not isinstance(raw, Mapping): + raise ProjectionValueError("task lease must contain an object") + if raw.get("goal_id") != goal_id or raw.get("todo_id") != file_stem: + raise ProjectionValueError("task lease identity does not match its shadow source") + return dict(canonical_value(dict(raw))) + + +def todo_partition_projection( + *, + handoff_mode: str, + todos: Iterable[object], +) -> dict[str, Any]: + """The state guarded by the goal's active-state file lock.""" + + compact = [item for item in (compact_todo(raw) for raw in todos) if item is not None] + compact.sort(key=lambda item: str(item["todo_id"])) + return {"handoff_mode": handoff_mode, "todos": compact} + + +def lease_partition_projection( + records: Iterable[tuple[str, object]], + *, + goal_id: str, +) -> dict[str, Any]: + """The state guarded by the goal's task-lease lock. + + ``records`` pairs each lease file stem with its decoded JSON object. + """ + + leases = [ + compact_lease(raw, goal_id=goal_id, file_stem=stem) + for stem, raw in sorted(records, key=lambda pair: pair[0]) + ] + return {"leases": leases} + + +def partition_digest(projection: Mapping[str, Any]) -> str: + return sha256_digest(dict(projection)) + + +def head_comparison_view(head: Mapping[str, Any]) -> dict[str, Any]: + """The part of a candidate head that parity compares against the source.""" + + return { + "handoff_mode": head.get("handoff_mode"), + "todos": head.get("todos"), + "leases": head.get("leases"), + } + + +def head_digest(head: Mapping[str, Any]) -> str: + return sha256_digest(head_comparison_view(head)) + + +__all__ = [ + "LEASE_PARTITION", + "LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V0", + "PARTITIONS", + "TODO_FIELDS", + "TODO_PARTITION", + "ProjectionValueError", + "canonical_bytes", + "canonical_value", + "compact_lease", + "compact_todo", + "head_comparison_view", + "head_digest", + "lease_partition_projection", + "partition_digest", + "sha256_digest", + "text_digest", + "todo_partition_projection", +] diff --git a/loopx/control_plane/coordination/runtime_shadow_writer_adapter.py b/loopx/control_plane/coordination/runtime_shadow_writer_adapter.py new file mode 100644 index 0000000000..023310d6a5 --- /dev/null +++ b/loopx/control_plane/coordination/runtime_shadow_writer_adapter.py @@ -0,0 +1,147 @@ +"""Transaction-capture orchestration for legacy Todo and lease writers.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from ...history import load_registry +from ...registry import find_registry_goal +from . import local_authority_shadow_outbox as outbox +from .local_authority_shadow_projection import LEASE_PARTITION +from .runtime_shadow import resolve_coordination_runtime_shadow_config + + +def begin_todo_runtime_shadow_capture( + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + state_path: Path, + write_class: str, + original_text: str, +) -> outbox.TodoPartitionCapture: + """Create the default-off transaction capture while the Todo lock is held.""" + + try: + registry = load_registry(registry_path) + goal = find_registry_goal(registry, goal_id) + enabled = resolve_coordination_runtime_shadow_config(goal).enabled + from ...rollout_event_log import load_rollout_events, rollout_event_log_path + from ..todos.todo_index import MAX_TODO_INDEX_ROLLOUT_EVENTS_PER_GOAL + from .local_authority_shadow_adapter import todo_partition_projector + + events = load_rollout_events( + rollout_event_log_path(runtime_root, goal_id), + limit=MAX_TODO_INDEX_ROLLOUT_EVENTS_PER_GOAL, + ) + projector = todo_partition_projector( + goal, + state_path=state_path, + rollout_events=events, + ) + except Exception: + enabled = False + projector = None + return outbox.TodoPartitionCapture.begin( + enabled=enabled, + runtime_root=runtime_root, + goal_id=goal_id, + state_path=state_path, + write_class=write_class, + original_text=original_text, + projector=projector, + ) + + +def settle_todo_runtime_shadow_capture( + payload: dict[str, Any], + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + write_class: str, + capture: outbox.TodoPartitionCapture, +) -> dict[str, Any]: + """Boundedly drain one transaction capture after releasing the Todo lock.""" + + from .local_authority_shadow_adapter import ( + capture_evidence, + drain_local_authority_shadow_outbox, + observe_todo_local_authority_commit, + ) + + drain = ( + drain_local_authority_shadow_outbox( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + if capture.outcome.entry_id is not None + else None + ) + payload["coordination_runtime_shadow"] = capture_evidence( + goal_id=goal_id, + capture=capture.outcome, + drain=drain, + ) + return observe_todo_local_authority_commit( + payload, + registry_path, + goal_id, + write_class, + runtime_root=runtime_root, + ) + + +def settle_lease_runtime_shadow_capture( + payload: dict[str, Any], + *, + registry_path: Path | None, + runtime_root: Path, + goal_id: str, +) -> dict[str, Any]: + """Turn the TS writer capture receipt into bounded runtime-shadow evidence.""" + + raw = payload.pop("coordination_runtime_shadow_capture", None) + if not isinstance(raw, Mapping) or registry_path is None: + return payload + capture = outbox.CaptureOutcome( + entry_id=str(raw["entry_id"]) if isinstance(raw.get("entry_id"), str) else None, + partition=LEASE_PARTITION, + seq=int(raw["seq"]) if isinstance(raw.get("seq"), int) else None, + source_bytes_digest=( + str(raw["source_bytes_digest"]) + if isinstance(raw.get("source_bytes_digest"), str) + else None + ), + failure=dict(raw["failure"]) if isinstance(raw.get("failure"), Mapping) else None, + ) + from .local_authority_shadow_adapter import ( + capture_evidence, + drain_local_authority_shadow_outbox, + ) + + drain = ( + drain_local_authority_shadow_outbox( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) + if capture.entry_id is not None + else None + ) + payload["coordination_runtime_shadow"] = capture_evidence( + goal_id=goal_id, + capture=capture, + drain=drain, + ) + return payload + + +__all__ = [ + "begin_todo_runtime_shadow_capture", + "settle_lease_runtime_shadow_capture", + "settle_todo_runtime_shadow_capture", +] diff --git a/loopx/control_plane/effect_runtime_handlers.ts b/loopx/control_plane/effect_runtime_handlers.ts index 9f14d6162b..22e33e28e5 100644 --- a/loopx/control_plane/effect_runtime_handlers.ts +++ b/loopx/control_plane/effect_runtime_handlers.ts @@ -98,7 +98,11 @@ import { executeTaskLeaseAcquire, } from "./work_items/task_lease_acquire.ts"; import { executeTaskLeaseLifecycle } from "./work_items/task_lease_lifecycle.ts"; -import { recordLocalAuthorityShadow } from "./coordination/local_authority_shadow.ts"; +import { + commitLocalAuthorityShadowEntry, + readLocalAuthorityShadow, + recordLocalAuthorityShadow, +} from "./coordination/local_authority_shadow.ts"; import { evaluateTaskLeaseLifecycleDecision } from "./work_items/task_lease_lifecycle_decision.ts"; import { bootstrapCoordinationRuntimeShadow, @@ -399,6 +403,8 @@ export function createEffectRuntimeHandlers( ["task_lease.write_scopes.overlap", evaluateTaskLeaseWriteScopesOverlap], ["quota.monitor_poll.commit", evaluateQuotaMonitorPollCommit], ["coordination.local_authority_shadow.record", recordLocalAuthorityShadow], + ["coordination.runtime_shadow.commit_entry", commitLocalAuthorityShadowEntry], + ["coordination.runtime_shadow.outbox_read", readLocalAuthorityShadow], [ "effect.program_from_ordered_steps", (params) => effectProgramFromOrderedSteps( diff --git a/loopx/control_plane/effect_runtime_io.ts b/loopx/control_plane/effect_runtime_io.ts index 64fffcadee..a2e70eeeb4 100644 --- a/loopx/control_plane/effect_runtime_io.ts +++ b/loopx/control_plane/effect_runtime_io.ts @@ -457,3 +457,39 @@ export async function appendJsonLine( await handle.close(); } } + +async function syncDirectoryForDurableWrite(directory: string): Promise { + if (process.platform === "win32") return; + const handle = await open(directory, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +/** + * Write JSON so that a crash cannot leave a torn or unlinked file behind: + * temp file in the same directory, fsync, atomic rename, directory fsync. + */ +export async function durableWriteJson( + path: string, + payload: JsonObject, +): Promise { + const directory = dirname(path); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`; + const handle = await open(temporary, "wx", 0o600); + try { + try { + await handle.writeFile(`${JSON.stringify(payload, null, 1)}\n`, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + await rename(temporary, path); + await syncDirectoryForDurableWrite(directory); + } finally { + await rm(temporary, { force: true }); + } +} diff --git a/loopx/control_plane/quota/monitor_poll.py b/loopx/control_plane/quota/monitor_poll.py index 1f85463c54..cdd43afee9 100644 --- a/loopx/control_plane/quota/monitor_poll.py +++ b/loopx/control_plane/quota/monitor_poll.py @@ -112,6 +112,7 @@ def _vision_wait_state(before: dict[str, Any]) -> dict[str, Any]: def _registry_due_monitor( *, registry_path: Path | None, + runtime_root: Path | None, goal_id: str, todo_id: str | None, target_key: str | None, @@ -122,6 +123,7 @@ def _registry_due_monitor( item = resolve_monitor_todo_item( registry_path=registry_path, goal_id=goal_id, + runtime_root=runtime_root, todo_id=todo_id, target_key=target_key, ) @@ -145,12 +147,14 @@ def _decision_packet( todo_id: str | None, target_key: str | None, registry_path: Path | None = None, + runtime_root: Path | None = None, authorized_due_monitor_poll: bool | None = None, ) -> dict[str, Any]: lane = _mapping(before.get("work_lane_contract")) due_candidates = _due_monitor_candidates(before) registry_due = _registry_due_monitor( registry_path=registry_path, + runtime_root=runtime_root, goal_id=goal_id, todo_id=todo_id, target_key=target_key, @@ -636,12 +640,22 @@ def record_quota_monitor_poll_for_decision( if normalized_turn_id else f"quota-monitor-poll:{goal_id}:{uuid.uuid4().hex}" ) + if execute and (safe_todo_id or safe_target_key): + from ..coordination.legacy_writer_fence import ( + require_legacy_coordination_write_allowed, + ) + + require_legacy_coordination_write_allowed( + runtime_root=runtime_root, + goal_id=goal_id, + ) decision = _decision_packet( before, goal_id=goal_id, todo_id=safe_todo_id, target_key=safe_target_key, registry_path=registry_path, + runtime_root=runtime_root, ) observation = _observation_packet( before=before, diff --git a/loopx/control_plane/scheduler/monitor_poll_writeback.py b/loopx/control_plane/scheduler/monitor_poll_writeback.py index f3a2d47117..f060c67209 100644 --- a/loopx/control_plane/scheduler/monitor_poll_writeback.py +++ b/loopx/control_plane/scheduler/monitor_poll_writeback.py @@ -115,6 +115,7 @@ def resolve_monitor_todo_item( *, registry_path: Path, goal_id: str, + runtime_root: Path | None = None, todo_id: str | None = None, target_key: str | None = None, ) -> dict[str, Any]: @@ -124,7 +125,12 @@ def resolve_monitor_todo_item( safe_target_key = str(target_key or "").strip() if not normalized_todo_id and not safe_target_key: raise ValueError("monitor todo writeback requires --todo-id or --target-key") - payload = list_goal_todos(registry_path=registry_path, goal_id=goal_id, role="agent") + payload = list_goal_todos( + registry_path=registry_path, + goal_id=goal_id, + role="agent", + runtime_root_arg=str(runtime_root) if runtime_root is not None else None, + ) items = payload.get("todos") if isinstance(payload.get("todos"), list) else [] if normalized_todo_id: matches = [ @@ -202,9 +208,17 @@ def write_monitor_poll_todo_state( """ from ...todos import add_goal_todo, update_goal_todo + from ..coordination.legacy_writer_fence import ( + require_legacy_coordination_write_allowed, + ) if not todo_id and not target_key: return None + if execute: + require_legacy_coordination_write_allowed( + runtime_root=runtime_root, + goal_id=goal_id, + ) safe_result_hash = str(result_hash or "").strip() if not safe_result_hash: raise ValueError("monitor todo writeback requires --result-hash") @@ -223,6 +237,7 @@ def write_monitor_poll_todo_state( item = resolve_monitor_todo_item( registry_path=registry_path, goal_id=goal_id, + runtime_root=runtime_root, todo_id=todo_id, target_key=target_key, ) diff --git a/loopx/control_plane/scheduler/state_transition_rules.py b/loopx/control_plane/scheduler/state_transition_rules.py index a615a574d6..516e5230f7 100644 --- a/loopx/control_plane/scheduler/state_transition_rules.py +++ b/loopx/control_plane/scheduler/state_transition_rules.py @@ -276,15 +276,10 @@ def decide_scheduler_backoff_state( state_acknowledges_rrule = result.get( "scheduler_state_acknowledges_current_rrule" ) - string_fields = { - key: result.get(key) - for key in ( - "current_rrule", - "last_applied_rrule", - "observed_host_rrule", - "effective_host_rrule", - ) - } + current_rrule = result.get("current_rrule") + last_applied_rrule = result.get("last_applied_rrule") + observed_result_rrule = result.get("observed_host_rrule") + effective_host_rrule = result.get("effective_host_rrule") recorded_failure = result.get("recorded_host_failure") if ( result.get("operation") != "backoff" @@ -300,7 +295,10 @@ def decide_scheduler_backoff_state( or not isinstance(repeated_failed_pair, bool) or not isinstance(current_rrule_applied, bool) or not isinstance(state_acknowledges_rrule, bool) - or any(not isinstance(value, str) for value in string_fields.values()) + or not isinstance(current_rrule, str) + or not isinstance(last_applied_rrule, str) + or not isinstance(observed_result_rrule, str) + or not isinstance(effective_host_rrule, str) or (recorded_failure is not None and not isinstance(recorded_failure, dict)) ): raise RuntimeError("TypeScript scheduler backoff result shape mismatch") @@ -319,10 +317,10 @@ def decide_scheduler_backoff_state( cadence=cadence, host=host, current_interval_minutes=current_interval, - current_rrule=string_fields["current_rrule"], - last_applied_rrule=string_fields["last_applied_rrule"], - observed_host_rrule=string_fields["observed_host_rrule"], - effective_host_rrule=string_fields["effective_host_rrule"], + current_rrule=current_rrule, + last_applied_rrule=last_applied_rrule, + observed_host_rrule=observed_result_rrule, + effective_host_rrule=effective_host_rrule, all_host_update_failures=_object_tuple( result.get("all_host_update_failures"), label="scheduler all-host failure cache", diff --git a/loopx/control_plane/todos/contract.py b/loopx/control_plane/todos/contract.py index 5ad391b761..0e6759c52b 100644 --- a/loopx/control_plane/todos/contract.py +++ b/loopx/control_plane/todos/contract.py @@ -11,7 +11,7 @@ from .completion_state import ( normalize_todo_completion_continuation, normalize_todo_completion_recovery, - normalize_todo_no_followup, + normalize_todo_no_followup as normalize_todo_no_followup, require_todo_completion_metadata, ) from .resume_condition import ( diff --git a/loopx/control_plane/todos/decision_scope.py b/loopx/control_plane/todos/decision_scope.py index 8490d78648..c12a604d78 100644 --- a/loopx/control_plane/todos/decision_scope.py +++ b/loopx/control_plane/todos/decision_scope.py @@ -380,7 +380,8 @@ def build_required_decision_scope_repair_hint( ) -> dict[str, Any] | None: if consistency.get("ok") is not False: return None - errors = consistency.get("errors") if isinstance(consistency.get("errors"), list) else [] + raw_errors = consistency.get("errors") + errors = raw_errors if isinstance(raw_errors, list) else [] missing_gate_scope = any( isinstance(error, dict) and error.get("reason_code") == "multi_agent_user_gate_missing_scope" diff --git a/loopx/control_plane/todos/deferred_resume.py b/loopx/control_plane/todos/deferred_resume.py index ae43a3a46d..10572a4e90 100644 --- a/loopx/control_plane/todos/deferred_resume.py +++ b/loopx/control_plane/todos/deferred_resume.py @@ -170,11 +170,13 @@ def todo_summary_deferred_items( ) -> list[dict[str, Any]]: if not isinstance(value, dict): return [] - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_source_items = value.get(key) + source_items = raw_source_items if isinstance(raw_source_items, list) else [] if not source_items and key == "deferred_items": + raw_items = value.get("items") source_items = [ item - for item in value.get("items", []) + for item in (raw_items if isinstance(raw_items, list) else []) if isinstance(item, dict) and todo_item_is_deferred(item) ] items: list[dict[str, Any]] = [] @@ -214,14 +216,14 @@ def _dedupe_todo_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: def todo_summary_resume_blocked_items(value: dict[str, Any]) -> list[dict[str, Any]]: if not isinstance(value, dict): return [] + raw_source_items = value.get("resume_blocked_items") source_items = ( - value.get("resume_blocked_items") - if isinstance(value.get("resume_blocked_items"), list) - else [] + list(raw_source_items) if isinstance(raw_source_items, list) else [] ) if not source_items: for key in ("items", "backlog_items", "first_open_items"): - raw_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_value_items = value.get(key) + raw_items = raw_value_items if isinstance(raw_value_items, list) else [] source_items.extend(item for item in raw_items if isinstance(item, dict)) items: list[dict[str, Any]] = [] for item in source_items: @@ -250,7 +252,8 @@ def _monitor_target_todo_ids(value: dict[str, Any]) -> set[str]: "backlog_items", "first_open_items", ): - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_items = value.get(key) + source_items = raw_items if isinstance(raw_items, list) else [] for item in source_items: if not isinstance(item, dict): continue @@ -270,11 +273,8 @@ def todo_summary_monitor_blocked_resume_items( for item in todo_summary_resume_blocked_items(value): if _todo_task_class(item) != TODO_TASK_CLASS_ADVANCEMENT: continue - condition = ( - item.get("resume_condition") - if isinstance(item.get("resume_condition"), dict) - else {} - ) + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} if condition.get("kind") == "monitor_changed": # This is the intentional typed wait lifecycle. The monitor stays # independently schedulable and its generation fence owns resume; @@ -334,11 +334,8 @@ def todo_summary_blocked_successor_items( if claimed_by and claimed_by != agent_id: continue resume_when = normalize_todo_resume_when(item.get("resume_when")) - condition = ( - item.get("resume_condition") - if isinstance(item.get("resume_condition"), dict) - else {} - ) + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} if not resume_when or condition.get("satisfied") is not False: continue target_task_class = normalize_todo_task_class( diff --git a/loopx/control_plane/todos/goal_todo_projection.py b/loopx/control_plane/todos/goal_todo_projection.py index 583a39473b..14f3e1abd5 100644 --- a/loopx/control_plane/todos/goal_todo_projection.py +++ b/loopx/control_plane/todos/goal_todo_projection.py @@ -364,12 +364,39 @@ def todo_summaries_from_fields( uncapped_todo_count=uncapped_todo_count, ) +def project_goal_todo_items( + goal: dict[str, Any] | None, + *, + state_text: str, + state_path: Path, + rollout_events: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Every user and agent todo item projected from one active-state text. + + Same items ``list_goal_todos`` returns without filters, computed from the + caller's text instead of the file so it can run inside the writer's lock. + """ + + return goal_todo_summaries( + goal, + state_text=state_text, + state_path=state_path, + rollout_events=rollout_events, + roles=["user", "agent"], + status=None, + todo_id=None, + agent_id=None, + limit=None, + ).todos + + __all__ = [ "GoalTodoSummaries", "empty_todo_summary", "filtered_todo_summary", "goal_todo_summaries", "merge_todo_projection_fields", + "project_goal_todo_items", "summary_items", "todo_summaries_from_fields", ] diff --git a/loopx/control_plane/todos/handoff_gate.py b/loopx/control_plane/todos/handoff_gate.py index 97465b3446..2d10511e8d 100644 --- a/loopx/control_plane/todos/handoff_gate.py +++ b/loopx/control_plane/todos/handoff_gate.py @@ -217,7 +217,8 @@ def todo_summary_handoff_gates(value: dict[str, Any]) -> list[dict[str, Any]]: projected = value.get("handoff_gates") if isinstance(projected, list): return [item for item in projected if isinstance(item, dict)] - source_items = value.get("items") if isinstance(value.get("items"), list) else [] + raw_items = value.get("items") + source_items = raw_items if isinstance(raw_items, list) else [] return build_todo_handoff_gate_states(source_items) diff --git a/loopx/control_plane/todos/projection.py b/loopx/control_plane/todos/projection.py index c16f0c4428..c6b570ed26 100644 --- a/loopx/control_plane/todos/projection.py +++ b/loopx/control_plane/todos/projection.py @@ -88,8 +88,9 @@ def todo_priority_rank(value: Any, *, text_mode: str = "label") -> int: def todo_index_rank(item: dict[str, Any]) -> int: + raw_index = item.get("index") try: - return int(item.get("index")) + return int(raw_index) if raw_index is not None else TODO_MISSING_INDEX except (TypeError, ValueError): return TODO_MISSING_INDEX @@ -722,11 +723,8 @@ def todo_summary_first_executable_item( ) -> dict[str, Any] | None: if not isinstance(summary, dict): return None - items = ( - summary.get("first_executable_items") - if isinstance(summary.get("first_executable_items"), list) - else [] - ) + raw_items = summary.get("first_executable_items") + items = raw_items if isinstance(raw_items, list) else [] for item in items: if not isinstance(item, dict): continue diff --git a/loopx/control_plane/todos/quota_summary.py b/loopx/control_plane/todos/quota_summary.py index 2184cdb5c1..96b2c957b4 100644 --- a/loopx/control_plane/todos/quota_summary.py +++ b/loopx/control_plane/todos/quota_summary.py @@ -287,10 +287,16 @@ def validate_todo_source_contract( key: _strict_non_negative_int(value.get(key)) for key in ("total_count", "open_count", "done_count", "deferred_count") } + total_count = counts["total_count"] + open_count = counts["open_count"] + done_count = counts["done_count"] + deferred_count = counts["deferred_count"] valid_counts = ( - all(count is not None for count in counts.values()) - and counts["total_count"] - == counts["open_count"] + counts["done_count"] + counts["deferred_count"] + total_count is not None + and open_count is not None + and done_count is not None + and deferred_count is not None + and total_count == open_count + done_count + deferred_count ) valid_proof = bool( isinstance(proof, dict) @@ -299,11 +305,12 @@ def validate_todo_source_contract( and proof.get("derived") is True and bool(str(value.get("source_section") or "").strip()) and type(proof.get("item_count")) is int - and proof.get("item_count") == counts["total_count"] + and proof.get("item_count") == total_count ) valid_terminal_closure = bool( valid_counts and valid_proof + and isinstance(proof, dict) and _terminal_closure_proof_is_valid( value, counts=counts, @@ -320,18 +327,25 @@ def validate_todo_source_contract( intent = value.get("closure_intent") terminal_proof = value.get("terminal_closure_proof") + intent_count = intent.get("count") if isinstance(intent, dict) else None valid_intent = bool( valid_terminal_closure and isinstance(intent, dict) and intent.get("schema_version") == "todo_closure_intent_v0" and intent.get("kind") == "no_followup" and intent.get("derived") is True - and type(intent.get("count")) is int - and 0 < intent.get("count") <= counts["done_count"] + and type(intent_count) is int + and done_count is not None + and 0 < intent_count <= done_count and isinstance(terminal_proof, dict) - and intent.get("count") == terminal_proof.get("no_followup_count") + and intent_count == terminal_proof.get("no_followup_count") + ) + closure_intent = ( + {**intent, "source": "todo_no_followup"} + if valid_intent and isinstance(intent, dict) + else None ) - return completeness, {**intent, "source": "todo_no_followup"} if valid_intent else None + return completeness, closure_intent def _build_quota_todo_lanes( diff --git a/loopx/control_plane/todos/route_continuation.py b/loopx/control_plane/todos/route_continuation.py index c5c07a201b..111e8e42ba 100644 --- a/loopx/control_plane/todos/route_continuation.py +++ b/loopx/control_plane/todos/route_continuation.py @@ -124,7 +124,8 @@ def todo_summary_route_continuation_candidates( "route_continuation_replan_candidates", "route_continuation_candidates", ): - raw_items = value.get(key) if isinstance(value.get(key), list) else [] + source = value.get(key) + raw_items = source if isinstance(source, list) else [] source_items.extend(item for item in raw_items if isinstance(item, dict)) source_items.extend( diff --git a/loopx/control_plane/todos/succession_warning.py b/loopx/control_plane/todos/succession_warning.py index 6adb3385a4..985630c365 100644 --- a/loopx/control_plane/todos/succession_warning.py +++ b/loopx/control_plane/todos/succession_warning.py @@ -152,7 +152,7 @@ def build_todo_succession_warning_lanes( ][:item_limit] count = warning.get("count", summary.get("completed_without_successor_count")) try: - count = max(0, int(count)) + count = max(0, int(count)) if count is not None else len(items) except (TypeError, ValueError): count = len(items) if count <= 0 and not items: @@ -193,18 +193,17 @@ def todo_succession_gap_items( if not isinstance(summary, dict): return [] - warning = ( - summary.get("todo_succession_warning") - if isinstance(summary.get("todo_succession_warning"), dict) - else {} - ) + raw_warning = summary.get("todo_succession_warning") + warning = raw_warning if isinstance(raw_warning, dict) else {} if warning and warning.get("reason_code") != TODO_SUCCESSION_WARNING_REASON_CODE: return [] + warning_items = warning.get("items") + completed_items = summary.get("completed_without_successor_items") source_items = ( - warning.get("items") - if isinstance(warning.get("items"), list) - else summary.get("completed_without_successor_items") - if isinstance(summary.get("completed_without_successor_items"), list) + warning_items + if isinstance(warning_items, list) + else completed_items + if isinstance(completed_items, list) else [] ) items = [item for item in source_items if isinstance(item, dict)] diff --git a/loopx/control_plane/todos/summary_item.py b/loopx/control_plane/todos/summary_item.py index 621bff566d..d180d6b20d 100644 --- a/loopx/control_plane/todos/summary_item.py +++ b/loopx/control_plane/todos/summary_item.py @@ -200,7 +200,8 @@ def todo_summary_source_items(value: dict[str, Any]) -> list[dict[str, Any]]: ready_successor_todo_ids = handoff_ready_successor_todo_ids(value) open_items: list[dict[str, Any]] = [] for key in TODO_SUMMARY_SOURCE_KEYS: - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_items = value.get(key) + source_items = raw_items if isinstance(raw_items, list) else [] for item in source_items: if not isinstance(item, dict) or item.get("done") is True: continue @@ -251,7 +252,8 @@ def todo_planning_source_items( planning_items: list[dict[str, Any]] = [] seen: set[str] = set() for key in TODO_PLANNING_SOURCE_KEYS: - source_items = value.get(key) if isinstance(value.get(key), list) else [] + raw_items = value.get(key) + source_items = raw_items if isinstance(raw_items, list) else [] for item in source_items: if not isinstance(item, dict) or ( not include_terminal and item.get("status") == "done" diff --git a/loopx/control_plane/todos/todo_summary.py b/loopx/control_plane/todos/todo_summary.py index 61a7f9ab82..ca4ad85d45 100644 --- a/loopx/control_plane/todos/todo_summary.py +++ b/loopx/control_plane/todos/todo_summary.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime import re from typing import Any, Callable, Optional @@ -628,19 +629,23 @@ def todo_item_is_actionable_open(item: dict[str, Any]) -> bool: return projection_todo_item_is_actionable_open(item) -def todo_item_next_due_at(item: dict[str, Any]): +def todo_item_next_due_at(item: dict[str, Any]) -> datetime | None: return projection_todo_item_next_due_at(item) -def todo_item_expires_at(item: dict[str, Any]): +def todo_item_expires_at(item: dict[str, Any]) -> datetime | None: return projection_todo_item_expires_at(item) -def todo_item_is_due_monitor(item: dict[str, Any], *, now=None) -> bool: +def todo_item_is_due_monitor( + item: dict[str, Any], *, now: datetime | None = None +) -> bool: return projection_todo_item_is_due_monitor(item, now=now, task_text_keys=("text",)) -def todo_item_missing_monitor_schedule(item: dict[str, Any], *, now=None) -> bool: +def todo_item_missing_monitor_schedule( + item: dict[str, Any], *, now: datetime | None = None +) -> bool: return projection_todo_item_missing_monitor_schedule(item, now=now, task_text_keys=("text",)) @@ -778,7 +783,8 @@ def dependency_blocker_summary( goal_id = str(item.get("goal_id") or "") if not goal_id or goal_id == current_goal_id: continue - user_todos = item.get("user_todos") if isinstance(item.get("user_todos"), dict) else {} + raw_user_todos = item.get("user_todos") + user_todos = raw_user_todos if isinstance(raw_user_todos, dict) else {} for todo in user_todos.get("items") or []: if not isinstance(todo, dict) or todo.get("done"): continue @@ -940,8 +946,9 @@ def todo_item_is_succession_tracked_completion(item: dict[str, Any]) -> bool: def _completed_succession_sort_key(item: dict[str, Any]) -> tuple[str, int]: + raw_index = item.get("index") try: - index = int(item.get("index")) + index = int(raw_index) if raw_index is not None else 0 except (TypeError, ValueError): index = 0 timestamp = str(item.get("updated_at") or item.get("completed_at") or "") @@ -1193,7 +1200,7 @@ def compact_todo_group( for item in lanes.open_items if normalize_todo_id(item.get("todo_id")) not in watch_only_ids ] - summary = { + summary: dict[str, Any] = { "schema_version": "todo_summary_v0", "source_section": source_section, "total_count": len(items), diff --git a/loopx/control_plane/todos/user_gate.py b/loopx/control_plane/todos/user_gate.py index 290bb7a90d..bcbf10eab3 100644 --- a/loopx/control_plane/todos/user_gate.py +++ b/loopx/control_plane/todos/user_gate.py @@ -162,9 +162,10 @@ def apply_scoped_user_gate_fallback_projection( if projected.get("effective_action") in {"skip", "monitor_quiet_skip", None}: projected["effective_action"] = "scoped_user_gate_fallback" + raw_execution_obligation = projected.get("execution_obligation") execution_obligation = ( - dict(projected.get("execution_obligation")) - if isinstance(projected.get("execution_obligation"), dict) + dict(raw_execution_obligation) + if isinstance(raw_execution_obligation, dict) else {} ) execution_obligation.update( @@ -199,9 +200,10 @@ def build_gate_prompt( question = str(item.get("operator_question") or "").strip() recommended_action = str(item.get("recommended_action") or "").strip() next_handoff_condition = str(item.get("next_handoff_condition") or "").strip() + raw_missing_gates = item.get("missing_gates") missing_gates = [ str(gate).strip() - for gate in (item.get("missing_gates") if isinstance(item.get("missing_gates"), list) else []) + for gate in (raw_missing_gates if isinstance(raw_missing_gates, list) else []) if str(gate).strip() ] if user_todo_summary is None: diff --git a/loopx/control_plane/work_items/autonomous_replan_obligation.py b/loopx/control_plane/work_items/autonomous_replan_obligation.py index 9cefced291..d9684708db 100644 --- a/loopx/control_plane/work_items/autonomous_replan_obligation.py +++ b/loopx/control_plane/work_items/autonomous_replan_obligation.py @@ -59,7 +59,7 @@ def with_replan_novelty_guidance(action: str) -> str: def replan_obligation_id_from_packet(value: Any) -> str | None: - packet = value if isinstance(value, Mapping) else {} + packet: Mapping[str, Any] = value if isinstance(value, Mapping) else {} return normalize_todo_replan_obligation_id(packet.get("obligation_id")) @@ -68,10 +68,9 @@ def todo_lifecycle_settlement_obligation( ) -> Mapping[str, Any] | None: """Return the lifecycle-settlement subtype from a status payload or obligation.""" - obligation = ( - value.get("autonomous_replan_obligation") - if isinstance(value.get("autonomous_replan_obligation"), Mapping) - else value + raw_obligation = value.get("autonomous_replan_obligation") + obligation: Mapping[str, Any] = ( + raw_obligation if isinstance(raw_obligation, Mapping) else value ) if obligation.get("resolution_mode") != TODO_LIFECYCLE_SETTLEMENT_RESOLUTION_MODE: return None @@ -91,13 +90,14 @@ def project_todo_lifecycle_settlement_reentry( obligation = todo_lifecycle_settlement_obligation(payload) if obligation is None: return None + raw_triggers = obligation.get("triggers") triggers = [ { key: item.get(key) for key in ("kind", "todo_id", "completion_turn_key") if item.get(key) is not None } - for item in obligation.get("triggers") or [] + for item in (raw_triggers if isinstance(raw_triggers, list) else []) if isinstance(item, Mapping) and item.get("kind") == "completed_advancement_without_successor" and normalize_todo_id(item.get("todo_id")) @@ -132,14 +132,14 @@ def build_autonomous_replan_cli_actions( ) if lifecycle_reentry is not None: return list(lifecycle_reentry["next_cli_actions"]) - packet = ( - payload.get("replan_action_packet") - if isinstance(payload.get("replan_action_packet"), Mapping) - else {} + raw_packet = payload.get("replan_action_packet") + packet: Mapping[str, Any] = ( + raw_packet if isinstance(raw_packet, Mapping) else {} ) - writeback_contract = ( - packet.get("writeback_contract") - if isinstance(packet.get("writeback_contract"), Mapping) + raw_writeback_contract = packet.get("writeback_contract") + writeback_contract: Mapping[str, Any] = ( + raw_writeback_contract + if isinstance(raw_writeback_contract, Mapping) else {} ) successor_command = str( @@ -391,7 +391,11 @@ def _monitor_no_change_evidence( if not isinstance(agent_todos, dict): return None raw_monitors = agent_todos.get("monitor_open_items") - monitors = [item for item in raw_monitors or [] if isinstance(item, dict)] + monitors = [ + item + for item in (raw_monitors if isinstance(raw_monitors, list) else []) + if isinstance(item, dict) + ] stalled: list[tuple[int, dict[str, Any]]] = [] for item in monitors: try: @@ -454,8 +458,9 @@ def _future_due_blocking_monitor( if not accountable_agent_id: return None raw_monitors = agent_todos.get("monitor_open_items") + monitor_items = raw_monitors if isinstance(raw_monitors, list) else [] monitors_by_id: dict[str, dict[str, Any]] = {} - for monitor in raw_monitors or []: + for monitor in monitor_items: if not isinstance(monitor, dict): continue monitor_todo_id = normalize_todo_id(monitor.get("todo_id")) @@ -475,11 +480,8 @@ def _future_due_blocking_monitor( claimed_by = normalize_todo_claimed_by(item.get("claimed_by")) if accountable_agent_id and claimed_by and claimed_by != accountable_agent_id: continue - condition = ( - item.get("resume_condition") - if isinstance(item.get("resume_condition"), dict) - else {} - ) + raw_condition = item.get("resume_condition") + condition = raw_condition if isinstance(raw_condition, dict) else {} monitor_todo_id = normalize_todo_id( item.get("blocking_monitor_todo_id") or condition.get("target_todo_id") diff --git a/loopx/control_plane/work_items/progress_observation.py b/loopx/control_plane/work_items/progress_observation.py index c8be9405ae..512521688e 100644 --- a/loopx/control_plane/work_items/progress_observation.py +++ b/loopx/control_plane/work_items/progress_observation.py @@ -449,10 +449,9 @@ def semantic_delta_from_writeback( path_outcome = "" if isinstance(agent_vision, Mapping): vision_state = normalize_goal_vision_state(agent_vision.get("state")) - path_delta = ( - agent_vision.get("path_delta") - if isinstance(agent_vision.get("path_delta"), Mapping) - else {} + raw_path_delta = agent_vision.get("path_delta") + path_delta: Mapping[str, Any] = ( + raw_path_delta if isinstance(raw_path_delta, Mapping) else {} ) path_outcome = str(path_delta.get("outcome") or "").strip() if vision_state != "no_followup" or path_outcome != "stop": @@ -463,15 +462,13 @@ def semantic_delta_from_writeback( ) if isinstance(agent_vision, Mapping): - patch = ( - agent_vision.get("vision_patch") - if isinstance(agent_vision.get("vision_patch"), Mapping) - else {} + raw_patch = agent_vision.get("vision_patch") + patch: Mapping[str, Any] = ( + raw_patch if isinstance(raw_patch, Mapping) else {} ) + raw_path_delta = agent_vision.get("path_delta") path_delta = ( - agent_vision.get("path_delta") - if isinstance(agent_vision.get("path_delta"), Mapping) - else {} + raw_path_delta if isinstance(raw_path_delta, Mapping) else {} ) path_outcome = str(path_delta.get("outcome") or "").strip() evidence_refs = [ @@ -629,17 +626,18 @@ def build_replan_action_packet( successor_priority = str(todo_action.get("priority") or "P1").strip() if successor_priority not in {"P0", "P1", "P2", "P3", "P4"}: successor_priority = "P1" - selected_gap = ( + raw_selected_gap = ( bounded_research_frontier.get("selected_gap") if isinstance(bounded_research_frontier, Mapping) - and isinstance(bounded_research_frontier.get("selected_gap"), Mapping) else None ) - raw_successor_binding = ( - selected_gap.get("successor_binding") - if isinstance(selected_gap, Mapping) - and isinstance(selected_gap.get("successor_binding"), Mapping) - else {} + selected_gap: Mapping[str, Any] | None = ( + raw_selected_gap if isinstance(raw_selected_gap, Mapping) else None + ) + selected_gap_values: Mapping[str, Any] = selected_gap or {} + raw_binding = selected_gap_values.get("successor_binding") + raw_successor_binding: Mapping[str, Any] = ( + raw_binding if isinstance(raw_binding, Mapping) else {} ) successor_binding = replan_successor_semantic_binding( action_kind=raw_successor_binding.get("action_kind"), @@ -650,9 +648,9 @@ def build_replan_action_packet( ) writeback_contract: dict[str, Any] = {} successor_summary = str( - (selected_gap or {}).get("successor_summary") or "" + selected_gap_values.get("successor_summary") or "" ).strip()[:240] - if isinstance(selected_gap, Mapping) and successor_summary and successor_binding: + if selected_gap is not None and successor_summary and successor_binding: safe_goal_id = _stable_id(goal_id, field="goal_id") or "" safe_agent_id = ( _stable_id(agent_id or obligation.get("agent_id"), field="agent_id") diff --git a/loopx/control_plane/work_items/project_asset.py b/loopx/control_plane/work_items/project_asset.py index 60264fbfdb..6b5d250f08 100644 --- a/loopx/control_plane/work_items/project_asset.py +++ b/loopx/control_plane/work_items/project_asset.py @@ -351,7 +351,8 @@ def project_asset_handoff_check_projection(item: dict[str, Any]) -> dict[str, An if not isinstance(project_asset, dict): return None - quota = project_asset.get("quota") if isinstance(project_asset.get("quota"), dict) else {} + raw_quota = project_asset.get("quota") + quota = raw_quota if isinstance(raw_quota, dict) else {} if not quota and isinstance(item.get("quota"), dict): quota = item["quota"] diff --git a/loopx/control_plane/work_items/task_lease.py b/loopx/control_plane/work_items/task_lease.py index f46d97eea2..dd977bc23d 100644 --- a/loopx/control_plane/work_items/task_lease.py +++ b/loopx/control_plane/work_items/task_lease.py @@ -428,7 +428,7 @@ def hold_task_lease_mutation_fence( def close(committed: bool, release_lease: bool) -> dict[str, Any] | None: return _execute_native_task_lease_lifecycle( runtime_root=runtime_root, - registry_path=None, + registry_path=registry_path, goal_id=normalized_goal_id, todo_id=normalized_todo_id, operation="fence_close", diff --git a/loopx/control_plane/work_items/task_lease_acquire.ts b/loopx/control_plane/work_items/task_lease_acquire.ts index 5be9f6e924..bf22eb563f 100644 --- a/loopx/control_plane/work_items/task_lease_acquire.ts +++ b/loopx/control_plane/work_items/task_lease_acquire.ts @@ -17,6 +17,11 @@ import { type JsonObject, } from "../effect_program.ts"; import { requireJsonObject } from "../runtime_decode.ts"; +import { + beginLeaseOutboxEntry, + decodeLocalAuthorityShadowBinding, + type LocalAuthorityShadowBinding, +} from "../coordination/local_authority_shadow_outbox.ts"; export const TASK_LEASE_ACQUIRE_REQUEST_SCHEMA_VERSION = "loopx_task_lease_acquire_native_v0"; @@ -74,6 +79,7 @@ interface AcquireRequest { ttl_seconds: number; expected_version: number | null; authority: AuthorityFacts; + runtime_shadow: LocalAuthorityShadowBinding | null; } export interface LeaseRecord extends JsonObject { @@ -441,6 +447,7 @@ function decodeRequest(value: unknown): AcquireRequest { ttl_seconds: normalizeTtl(request.ttl_seconds), expected_version: optionalInteger(request.expected_version, "expected_version"), authority, + runtime_shadow: decodeLocalAuthorityShadowBinding(request.runtime_shadow), }; } @@ -1336,8 +1343,29 @@ async function commitAcquire( }; await dependencies.beforeWrite?.(lease); await revalidateAuthoritySources(request.authority.source_receipts); + const shadowCapture = request.runtime_shadow === null + ? null + : await beginLeaseOutboxEntry({ + runtime_root: request.runtime_root, + goal_id: request.goal_id, + lease_directory: taskLeaseDirectory(request), + write_class: "task_lease_acquire", + operation_id: request.idempotency_key, + previous_lease: existing, + planned_lease: lease, + }); await atomicWriteJson(leasePath, lease); - return successEnvelope(request, lease, leasePath, acquireEffectId(request), false); + await shadowCapture?.commit(); + const response = successEnvelope(request, lease, leasePath, acquireEffectId(request), false); + if (shadowCapture !== null) { + response.coordination_runtime_shadow_capture = { + entry_id: shadowCapture.entry_id, + seq: shadowCapture.seq, + source_bytes_digest: shadowCapture.source_bytes_digest, + failure: shadowCapture.failure, + }; + } + return response; } export async function executeTaskLeaseAcquire( diff --git a/loopx/control_plane/work_items/task_lease_acquire_adapter.py b/loopx/control_plane/work_items/task_lease_acquire_adapter.py index e277030df8..42c4efc4e4 100644 --- a/loopx/control_plane/work_items/task_lease_acquire_adapter.py +++ b/loopx/control_plane/work_items/task_lease_acquire_adapter.py @@ -11,6 +11,7 @@ from ...history import load_registry from ...paths import resolve_runtime_root +from ..coordination.runtime_shadow import resolve_coordination_runtime_shadow_config from ..goals.active_state_event_projection import ( state_event_log_candidates as _state_event_log_candidates, ) @@ -344,6 +345,16 @@ def _finalize_native_acquire_result( result["handoff_mode"] = authority.get("handoff_mode") or HANDOFF_MODE_LEGACY result.pop("settlement", None) if result.get("ok") is True and result.get("acquired") is True: + from ..coordination.runtime_shadow_writer_adapter import ( + settle_lease_runtime_shadow_capture, + ) + + result = settle_lease_runtime_shadow_capture( + result, + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=goal_id, + ) result = _attach_local_authority_shadow( result, registry_path=registry_path, @@ -390,6 +401,13 @@ def execute_native_task_lease_acquire( "expected_version": expected_version, "authority": authority, } + registry = load_registry(registry_path) + goal = _registry_goal(registry, str(goal_id)) + if resolve_coordination_runtime_shadow_config(goal).enabled: + request["runtime_shadow"] = { + "schema_version": "loopx_coordination_runtime_shadow_binding_v0", + "provider": "file_v0", + } payload = _require_native_acquire_shape( effect_runtime_result("task_lease.acquire.native", request, timeout=15.0) ) @@ -630,6 +648,14 @@ def execute_native_task_lease_lifecycle( # CLI argument or persisted authority fact. "current_time": _now.isoformat() if _now is not None else None, } + if registry_path is not None: + registry = load_registry(registry_path) + goal = _registry_goal(registry, str(goal_id)) + if resolve_coordination_runtime_shadow_config(goal).enabled: + request["runtime_shadow"] = { + "schema_version": "loopx_coordination_runtime_shadow_binding_v0", + "provider": "file_v0", + } compacted_todo = _compact_lifecycle_todo(todo, todo_id=str(todo_id)) if compacted_todo is not None: request["todo"] = compacted_todo @@ -660,6 +686,17 @@ def execute_native_task_lease_lifecycle( result["handoff_mode"] = authority["handoff_mode"] if _legacy_provider_projection: result.pop("settlement", None) + if "coordination_runtime_shadow_capture" in result: + from ..coordination.runtime_shadow_writer_adapter import ( + settle_lease_runtime_shadow_capture, + ) + + result = settle_lease_runtime_shadow_capture( + result, + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=str(goal_id), + ) committed_mutation = ( normalized_operation == "renew" and result.get("renewed") is True ) or ( diff --git a/loopx/control_plane/work_items/task_lease_lifecycle.ts b/loopx/control_plane/work_items/task_lease_lifecycle.ts index ee414ca6ba..4764004143 100644 --- a/loopx/control_plane/work_items/task_lease_lifecycle.ts +++ b/loopx/control_plane/work_items/task_lease_lifecycle.ts @@ -58,6 +58,12 @@ import { type TaskLeaseLifecycleDecision, type TaskLeaseLifecycleDecisionInput, } from "./task_lease_lifecycle_decision.ts"; +import { + beginLeaseOutboxEntry, + decodeLocalAuthorityShadowBinding, + type LeaseOutboxCapture, + type LocalAuthorityShadowBinding, +} from "../coordination/local_authority_shadow_outbox.ts"; export const TASK_LEASE_LIFECYCLE_REQUEST_SCHEMA_VERSION = "loopx_task_lease_lifecycle_native_v0"; @@ -104,6 +110,7 @@ interface LifecycleRequest { fence_operation_id: string | null; current_time: Date | null; owner_pid: number | null; + runtime_shadow: LocalAuthorityShadowBinding | null; } interface LifecycleDependencies { @@ -471,6 +478,41 @@ function decodeRequest(value: unknown): LifecycleRequest { fence_operation_id: fenceOperationId, current_time: optionalDate(input.current_time, "current_time"), owner_pid: optionalPositiveInteger(input.owner_pid, "owner_pid"), + runtime_shadow: decodeLocalAuthorityShadowBinding(input.runtime_shadow), + }; +} + +async function captureLeaseWrite( + request: LifecycleRequest, + previous: LeaseRecord | null, + next: LeaseRecord, + writeClass: string, +): Promise> | null> { + if (request.runtime_shadow === null) return null; + return await beginLeaseOutboxEntry({ + runtime_root: request.runtime_root, + goal_id: request.goal_id, + lease_directory: taskLeaseDirectory(request), + write_class: writeClass, + operation_id: request.idempotency_key ?? request.fence_operation_id, + previous_lease: previous, + planned_lease: next, + }); +} + +function attachRuntimeShadowCapture( + response: JsonObject, + capture: LeaseOutboxCapture | null, +): JsonObject { + if (capture === null) return response; + return { + ...response, + coordination_runtime_shadow_capture: { + entry_id: capture.entry_id, + seq: capture.seq, + source_bytes_digest: capture.source_bytes_digest, + failure: capture.failure, + }, }; } @@ -1673,9 +1715,13 @@ async function ordinaryOperation( const response = responseForOrdinary(request, leasePath, next, false, { released: true }); await persistOperationReceipt(request, "prepared", next, response); await dependencies.beforeWrite?.(next); + const shadowCapture = await captureLeaseWrite( + request, existing, next, "task_lease_release", + ); await atomicWriteJson(leasePath, next); + await shadowCapture?.commit(); await persistOperationReceipt(request, "committed", next, response); - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } if (existing === null || decision.next_lease === null) { throw transitionError(request, "invalid_lease_snapshot", existing, leasePath); @@ -1704,9 +1750,13 @@ async function ordinaryOperation( await persistOperationReceipt(request, "prepared", next, response); await dependencies.beforeWrite?.(next); if (request.authority) await revalidateAuthoritySources(request.authority.source_receipts); + const shadowCapture = await captureLeaseWrite( + request, existing, next, `task_lease_${request.operation}`, + ); await atomicWriteJson(leasePath, next); + await shadowCapture?.commit(); await persistOperationReceipt(request, "committed", next, response); - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } function fencePayload( @@ -2212,7 +2262,11 @@ async function fenceVerify( }; await dependencies.beforeWrite?.(lease); await revalidateAuthoritySources(request.authority.source_receipts); + const shadowCapture = await captureLeaseWrite( + request, rawLease, lease, "task_lease_auto_acquire", + ); await atomicWriteJson(leasePath, lease); + await shadowCapture?.commit(); const response = fencePayload( { ...request, owner: request.owner }, lease, @@ -2231,7 +2285,7 @@ async function fenceVerify( base: receipt, }); keep = true; - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } // A response can be lost after an auto-acquire write but before the // receipt reaches the caller. The durable fence receipt proves that the @@ -2558,6 +2612,7 @@ async function fenceClose( if (!claimedOwner || claimedOwner.token !== request.lock_token) { throw new EffectRuntimeConflictError("task lease fence token is no longer held", "fence_token_invalid"); } + let shadowCapture: LeaseOutboxCapture | null = null; if (request.committed && request.release_lease) { const at = lifecycleNow(request, dependencies); const leasePath = leasePathFor(request); @@ -2631,7 +2686,11 @@ async function fenceClose( if (!alreadyReleased) { const next = releasedLease(lease, at); await dependencies.beforeWrite?.(next); + shadowCapture = await captureLeaseWrite( + request, lease, next, "task_lease_fence_close", + ); await atomicWriteJson(leasePath, next); + await shadowCapture?.commit(); released = true; } } @@ -2657,7 +2716,7 @@ async function fenceClose( fenceExpectedLeaseEpoch: request.fence_expected_lease_epoch ?? receipt?.fence_expected_lease_epoch, closeRequestDigest: fenceCloseRequestDigest(request), }); - return response; + return attachRuntimeShadowCapture(response, shadowCapture); } finally { if (claim) { await releaseFileMutationLock( @@ -2682,7 +2741,18 @@ export async function executeTaskLeaseLifecycle( if (request.operation === "fence_close") return await fenceClose(request, dependencies); if (request.operation === "terminal_verify" || request.operation === "holder_verify") { const fence = await fenceVerify(request, dependencies); - return { ok: true, schema_version: "task_lease_v0", action: request.operation, fence, settlement: lifecycleSettlement(request, "committed") }; + const capture = fence.coordination_runtime_shadow_capture; + delete fence.coordination_runtime_shadow_capture; + return { + ok: true, + schema_version: "task_lease_v0", + action: request.operation, + fence, + settlement: lifecycleSettlement(request, "committed"), + ...(capture === undefined + ? {} + : { coordination_runtime_shadow_capture: capture }), + }; } return await withFileMutationLock( legacyCoordinationLeaseLockPath(request.runtime_root, request.goal_id), diff --git a/loopx/help_surface.py b/loopx/help_surface.py index 343f68bd1e..cc6e142d60 100644 --- a/loopx/help_surface.py +++ b/loopx/help_surface.py @@ -288,6 +288,7 @@ MANPAGE_COMMAND_HELP_ONLY = frozenset( { "archive-runtime", + "authority-shadow", "backup-state", "capability", "chat-endpoint", diff --git a/loopx/todos.py b/loopx/todos.py index c72dcd6728..8fff3fbf25 100644 --- a/loopx/todos.py +++ b/loopx/todos.py @@ -128,9 +128,10 @@ enter_todo_ownership_handoff_gate, resolve_todo_completion_handoff, ) -from .control_plane.coordination.local_authority_shadow_adapter import ( - effective_runtime_root, - observe_todo_local_authority_commit as _shadow_todo, +from .control_plane.coordination.local_authority_shadow_adapter import effective_runtime_root +from .control_plane.coordination.runtime_shadow_writer_adapter import ( + begin_todo_runtime_shadow_capture, + settle_todo_runtime_shadow_capture, ) from .control_plane.work_items.task_lease import ( enter_terminal_todo_lease_fence, @@ -765,6 +766,11 @@ def add_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as handoff_gate_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_add", original_text=original, + ) lines = original.splitlines() updated_at = now_local() effective_claimed_by = ( @@ -921,7 +927,9 @@ def add_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() payload = { "ok": True, @@ -974,7 +982,10 @@ def add_goal_todo( write_class="todo_add", state_text=original, ) - return _shadow_todo(payload, registry_path, goal_id, "todo_add", runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + payload, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_add", capture=shadow_capture, + ) def resolve_todo_state( @@ -1085,6 +1096,11 @@ def update_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as handoff_gate_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_update", original_text=original, + ) lines = original.splitlines() updated_at = now_local() effective_claimed_by = ( @@ -1402,7 +1418,9 @@ def update_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() write_class = "todo_claim" if claim_only else "todo_update" payload = { "ok": True, @@ -1435,7 +1453,10 @@ def update_goal_todo( write_class=write_class, state_text=original, ) - return _shadow_todo(payload, registry_path, goal_id, write_class, runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + payload, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class=write_class, capture=shadow_capture, + ) def complete_goal_todo( @@ -1514,6 +1535,11 @@ def complete_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as lease_fence_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_complete", original_text=original, + ) lines = original.splitlines() updated_at = now_local() completion_match, completion_todo, event_context = ( @@ -1668,9 +1694,15 @@ def complete_goal_todo( task_lease_fence, committed=bool(event_result.get("changed")) and not dry_run, ) - write_class = "todo_complete_event_projection" - return _shadow_todo( - event_result, registry_path, goal_id, write_class, runtime_root=shadow_runtime_root + # This branch can append multiple state-log events inside the + # event writer. Capturing after that call would be observation, + # not a transaction-bound prepare/commit pair. Keep the gap + # explicit until the event writer owns the outbox boundary. + shadow_capture.skip("event_log_writer_not_bound") + return settle_todo_runtime_shadow_capture( + event_result, registry_path=registry_path, + runtime_root=shadow_runtime_root, goal_id=goal_id, + write_class="todo_complete_event_projection", capture=shadow_capture, ) if not isinstance(completion_state, dict): raise RuntimeError( @@ -1801,7 +1833,9 @@ def complete_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() release_verified_task_lease_fence( task_lease_fence, committed=changed and not dry_run, @@ -1829,7 +1863,10 @@ def complete_goal_todo( if effective_decision_outcome: result["decision_outcome"] = effective_decision_outcome result["self_merged"] = effective_self_merged - return _shadow_todo(result, registry_path, goal_id, "todo_complete", runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + result, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_complete", capture=shadow_capture, + ) def supersede_goal_todo( *, @@ -1875,6 +1912,11 @@ def supersede_goal_todo( runtime_root=shadow_runtime_root, ), ExitStack() as lease_fence_stack: original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_supersede", original_text=original, + ) lines = original.splitlines() updated_at = now_local() current_match = find_todo_block(lines, todo_id=todo_id, role=role) @@ -2026,7 +2068,9 @@ def supersede_goal_todo( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() release_verified_task_lease_fence(task_lease_fence, committed=changed and not dry_run) result = { "ok": True, @@ -2042,7 +2086,10 @@ def supersede_goal_todo( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } - return _shadow_todo(result, registry_path, goal_id, "todo_supersede", runtime_root=shadow_runtime_root) + return settle_todo_runtime_shadow_capture( + result, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_supersede", capture=shadow_capture, + ) def archive_completed_todos( @@ -2073,6 +2120,11 @@ def archive_completed_todos( runtime_root=shadow_runtime_root, ): original = resolved_state_file.read_text(encoding="utf-8") + shadow_capture = begin_todo_runtime_shadow_capture( + registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, state_path=resolved_state_file, + write_class="todo_archive_completed", original_text=original, + ) lines = original.splitlines() archive_result = archive_completed_todo_lines( lines, @@ -2087,7 +2139,9 @@ def archive_completed_todos( if changed: new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: + shadow_capture.prepare(new_text) resolved_state_file.write_text(new_text, encoding="utf-8") + shadow_capture.committed() result = { "ok": True, @@ -2098,6 +2152,7 @@ def archive_completed_todos( "project": str(resolved_project) if resolved_project else None, "updated_at": updated_at if changed else None, } - return _shadow_todo( - result, registry_path, goal_id, "todo_archive_completed", runtime_root=shadow_runtime_root + return settle_todo_runtime_shadow_capture( + result, registry_path=registry_path, runtime_root=shadow_runtime_root, + goal_id=goal_id, write_class="todo_archive_completed", capture=shadow_capture, ) diff --git a/pyproject.toml b/pyproject.toml index 5880ca09a7..6f5725ebe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,8 @@ files = [ "loopx/control_plane/coordination/executor.py", "loopx/control_plane/coordination/file_provider.py", "loopx/control_plane/coordination/head.py", + "loopx/control_plane/coordination/local_authority_shadow_outbox.py", + "loopx/control_plane/coordination/local_authority_shadow_projection.py", "loopx/control_plane/quota/effect_program.py", "loopx/control_plane/quota/states.py", "loopx/control_plane/runtime/event_store_migration_bridge.py", diff --git a/tests/control_plane/test_local_authority_shadow_cli_e2e.py b/tests/control_plane/test_local_authority_shadow_cli_e2e.py index 8ab7cccd0d..68fe35dbd8 100644 --- a/tests/control_plane/test_local_authority_shadow_cli_e2e.py +++ b/tests/control_plane/test_local_authority_shadow_cli_e2e.py @@ -467,3 +467,58 @@ def test_product_cli_runtime_root_override_keeps_one_candidate_lineage( assert not (registry_runtime / "goals" / goal_id / "task-leases").exists() assert store_path.is_relative_to(override_runtime) assert todo_id in state.read_text(encoding="utf-8") + + +def test_product_cli_authority_shadow_status_and_drain_read_without_creating_lineage( + tmp_path: Path, +) -> None: + goal_id = "shadow-cli-drain" + registry, _state, runtime_root = _workspace(tmp_path, goal_id=goal_id) + + default_off = _cli(registry, runtime_root, "authority-shadow", "status", "--goal-id", goal_id) + assert default_off["ok"] is True + assert default_off["schema_version"] == "loopx_authority_shadow_cli_v0" + assert default_off["config"] == {"enabled": False, "mode": None, "status": "disabled"} # type: ignore[index] + assert default_off["candidate"]["status"] == "missing" # type: ignore[index] + assert default_off["store_bytes"] == 0 + assert not (runtime_root / "authority-shadow").exists() + idle = _cli(registry, runtime_root, "authority-shadow", "drain", "--goal-id", goal_id) + assert idle["ok"] is True + assert idle["outcome"] == "nothing_pending" + assert idle["drained_count"] == 0 + assert idle["config_enabled"] is False + assert not (runtime_root / "authority-shadow").exists() + + _cli( + registry, + runtime_root, + "configure-goal", + "--goal-id", + goal_id, + "--local-authority-shadow-file", + "--execute", + ) + observed = _add_todo(registry, runtime_root, goal_id=goal_id, text="Observed through the v0 path.") + assert observed["authority_shadow"]["outcome"] == "captured" # type: ignore[index] + + status = _cli(registry, runtime_root, "authority-shadow", "status", "--goal-id", goal_id) + assert status["ok"] is True + assert status["config"]["status"] == "enabled" # type: ignore[index] + assert status["outbox"]["todos"]["committed_pending"] == 0 # type: ignore[index] + assert status["outbox"]["leases"]["prepared_only"] == 0 # type: ignore[index] + candidate = status["candidate"] + assert candidate["status"] == "loaded" # type: ignore[index] + assert candidate["cursor"] == "1" # type: ignore[index] + assert candidate["head_schema_version"] == "loopx_local_authority_shadow_projection_v0" # type: ignore[index] + assert candidate["codec_agreement"] is True # type: ignore[index] + assert candidate["partitions"] == {"todos": None, "leases": None} # type: ignore[index] + assert status["store_bytes"] > 0 + assert status["retention_pressure"] is False + assert str(runtime_root) not in json.dumps(status) + + drained = _cli(registry, runtime_root, "authority-shadow", "drain", "--goal-id", goal_id) + assert drained["ok"] is True + assert drained["outcome"] == "nothing_pending" + assert drained["config_enabled"] is True + _store_path, store = _store_document(runtime_root, goal_id) + assert store["cursor"] == "1" diff --git a/tests/control_plane/test_local_authority_shadow_drain.py b/tests/control_plane/test_local_authority_shadow_drain.py new file mode 100644 index 0000000000..73b760d4e6 --- /dev/null +++ b/tests/control_plane/test_local_authority_shadow_drain.py @@ -0,0 +1,615 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter +from loopx.control_plane.coordination import local_authority_shadow_outbox as outbox +from loopx.control_plane.coordination.local_authority_shadow_projection import ( + head_digest, + partition_digest, + text_digest, +) +from loopx.file_lock import exclusive_file_lock +from loopx.history import load_registry +from loopx.registry import find_registry_goal +from loopx.todos import add_goal_todo, list_goal_todos + + +GOAL_ID = "goal-drain" + + +def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-03T00:00:00+00:00\n" + "---\n\n" + "## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": ["agent-a"], + }, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def _record_todo_write( + registry: Path, + state: Path, + runtime_root: Path, + text: str, + *, + mark_committed: bool = True, + write_file: bool = True, +) -> outbox.TodoPartitionCapture: + """Simulate a hooked writer: capture around one add_goal_todo write.""" + + original = state.read_text(encoding="utf-8") + goal = find_registry_goal(load_registry(registry), GOAL_ID) + capture = outbox.TodoPartitionCapture.begin( + enabled=True, + runtime_root=runtime_root, + goal_id=GOAL_ID, + state_path=state, + write_class="todo_add", + original_text=original, + projector=adapter.todo_partition_projector(goal, state_path=state), + ) + if write_file: + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text=text, + task_class="advancement_task", + ) + assert result["ok"] is True + new_text = state.read_text(encoding="utf-8") + else: + # The bytes the writer would have produced, without producing them. + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text=text, + task_class="advancement_task", + ) + assert result["ok"] is True + new_text = state.read_text(encoding="utf-8") + state.write_text(original, encoding="utf-8") + capture.prepare(new_text) + assert capture.outcome.failure is None + assert capture.outcome.entry_id is not None + if mark_committed: + capture.committed() + return capture + + +def _drain(registry: Path, runtime_root: Path, **overrides: object) -> adapter.DrainResult: + return adapter.drain_local_authority_shadow_outbox( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + **overrides, # type: ignore[arg-type] + ) + + +def _todo_dir(runtime_root: Path) -> Path: + return outbox.partition_directory(runtime_root, GOAL_ID, "todos") + + +def test_drain_delivers_each_committed_entry_once_in_order_and_verifies_readback(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + captures = [ + _record_todo_write(registry, state, runtime_root, f"Fact {index}") for index in range(3) + ] + assert [capture.outcome.seq for capture in captures] == [1, 2, 3] + + result = _drain(registry, runtime_root) + + assert result.ok is True + assert result.outcome == "drained" + assert result.config_enabled is False + assert (result.delivered, result.replayed, result.no_op) == (3, 0, 0) + assert result.pending_after == 0 + assert result.prepared_only_after == 0 + assert result.budget_exhausted is False + assert result.candidate_readback_verified is True + assert result.last_cursor == "3" + assert (result.cursor_before, result.cursor_after, result.drained_count) == (None, "3", 3) + payload = result.to_payload() + assert payload["ok"] is True + assert payload["drained_count"] == 3 + assert payload["cursor_after"] == "3" + assert [item["outcome"] for item in result.entries] == ["delivered"] * 3 + assert [item["cursor"] for item in result.entries] == ["1", "2", "3"] + assert [item["entry_id"] for item in result.entries] == [ + capture.outcome.entry_id for capture in captures + ] + assert result.store_identity is not None + assert result.store_identity.startswith("file:") + assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] + cursor = outbox.read_cursor(_todo_dir(runtime_root)) + assert cursor is not None + assert cursor["last_seq"] == 3 + assert cursor["last_entry_id"] == captures[-1].outcome.entry_id + assert cursor["last_partition_digest"] == captures[-1].outcome.partition_digest + + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) + assert view["status"] == "loaded" + head = view["head"] + assert head["schema_version"] == "loopx_coordination_runtime_shadow_projection_v0" + assert head["partitions"]["todos"] == { + "seq": 3, + "partition_digest": captures[-1].outcome.partition_digest, + } + assert head["partitions"]["leases"] is None + listed = list_goal_todos(registry_path=registry, goal_id=GOAL_ID, runtime_root_arg=str(runtime_root)) + assert [todo["todo_id"] for todo in head["todos"]] == sorted( + todo["todo_id"] for todo in listed["todos"] + ) + assert view["head_digest"] == head_digest(head) == result.head_digest + assert [tx["operation_id"] for tx in view["scan"]["transactions"]] == [ + capture.outcome.entry_id for capture in captures + ] + receipts = [tx["receipts"][0] for tx in view["scan"]["transactions"]] + assert all(receipt["source_transaction_correlated"] is True for receipt in receipts) + assert all(receipt["durable_source_outbox"] is True for receipt in receipts) + assert all(receipt["parity_verdict"] == "not_evaluated" for receipt in receipts) + assert [receipt["source_bytes_digest"] for receipt in receipts] == [ + capture.outcome.source_bytes_digest for capture in captures + ] + assert str(runtime_root) not in json.dumps(view) + + again = _drain(registry, runtime_root) + assert again.outcome == "nothing_pending" + assert again.ok is True + + +def test_drain_replays_when_store_committed_but_cursor_was_not_written( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Crash after store commit") + + def crash(*_args: object, **_kwargs: object) -> None: + raise OSError("simulated crash before the drain cursor landed") + + monkeypatch.setattr(outbox, "write_cursor", crash) + first = _drain(registry, runtime_root) + assert first.outcome == "stopped" + assert first.reason_code == "shadow_drain_failed" + assert first.pending_after == 1 + monkeypatch.undo() + + second = _drain(registry, runtime_root) + assert second.ok is True + assert (second.delivered, second.replayed) == (0, 1) + assert second.entries[0]["entry_id"] == capture.outcome.entry_id + assert second.entries[0]["cursor"] == "1" + assert second.pending_after == 0 + assert second.candidate_readback_verified is True + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) + assert len(view["scan"]["transactions"]) == 1 + + +def test_drain_defers_when_another_drainer_holds_the_lock(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + _record_todo_write(registry, state, runtime_root, "Pending behind a drainer") + with exclusive_file_lock( + outbox.drain_lock_target(runtime_root, GOAL_ID), timeout_seconds=1.0, operation="test_hold" + ): + result = _drain(registry, runtime_root, lock_timeout_seconds=0.05) + assert result.outcome == "drain_deferred" + assert result.reason_code == "drain_lock_busy" + assert result.ok is False + assert result.pending_after == 1 + assert not (runtime_root / "authority-shadow" / "file").exists() + + +def test_drain_batch_is_bounded_and_reports_what_it_left(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + for index in range(3): + _record_todo_write(registry, state, runtime_root, f"Bounded {index}") + + first = _drain(registry, runtime_root, max_entries=2) + assert first.outcome == "drained" + assert first.ok is True + assert first.delivered == 2 + assert first.budget_exhausted is True + assert first.pending_after == 1 + assert first.candidate_readback_verified is True + + second = _drain(registry, runtime_root) + assert second.delivered == 1 + assert second.pending_after == 0 + assert (second.cursor_before, second.cursor_after) == ("2", "3") + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID) + assert view["cursor"] == "3" + assert view["head"]["partitions"]["todos"]["seq"] == 3 + + +def test_drain_stops_in_order_when_the_store_boundary_misbehaves( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + for index in range(3): + _record_todo_write(registry, state, runtime_root, f"Ordered {index}") + real = adapter.effect_runtime_result + calls: list[str] = [] + + def flaky(method: str, params: object, **kwargs: object) -> object: + calls.append(method) + if method == "coordination.runtime_shadow.commit_entry" and len(calls) == 2: + return {"schema_version": "garbage"} + return real(method, params, **kwargs) + + monkeypatch.setattr(adapter, "effect_runtime_result", flaky) + result = _drain(registry, runtime_root) + assert result.outcome == "stopped" + assert result.delivered == 1 + assert result.stopped_at is not None + assert result.stopped_at["seq"] == 2 + assert result.stopped_at["reason_code"] == "shadow_commit_entry_result_invalid" + assert result.pending_after == 2 + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [2, 3] + monkeypatch.undo() + + recovered = _drain(registry, runtime_root) + assert recovered.delivered == 2 + assert recovered.pending_after == 0 + + +def test_prepared_only_entries_resolve_only_under_a_free_primary_lock(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + proven = _record_todo_write(registry, state, runtime_root, "Marker lost after write", mark_committed=False) + with exclusive_file_lock(state, timeout_seconds=1.0, operation="writer_in_flight"): + busy = _drain(registry, runtime_root) + assert busy.outcome == "drained" + assert busy.in_flight_partitions == ["todos"] + assert busy.prepared_only_after == 1 + assert busy.delivered == 0 + + result = _drain(registry, runtime_root) + assert result.ok is True + assert result.delivered == 1 + assert result.no_op == 0 + assert result.entries[0]["resolution"] == "committed_proven_by_readback" + assert result.entries[0]["entry_id"] == proven.outcome.entry_id + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + assert view["head"]["partitions"]["todos"]["partition_digest"] == proven.outcome.partition_digest + receipt = view["scan"]["transactions"][0]["receipts"][0] + assert receipt["resolution"] == "committed_proven_by_readback" + + abandoned = _record_todo_write( + registry, state, runtime_root, "Never landed", mark_committed=False, write_file=False + ) + result = _drain(registry, runtime_root) + assert result.delivered == 1 + assert result.no_op == 1 + assert result.entries[0]["resolution"] == "abandoned" + assert result.entries[0]["entry_id"] == abandoned.outcome.entry_id + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + assert view["cursor"] == "2" + assert view["head"]["partitions"]["todos"]["seq"] == 1 + assert view["scan"]["transactions"][1]["events"][0]["kind"] == "source_transaction_abandoned" + + +def test_unexplained_prepared_only_entry_triggers_reseed_under_the_primary_lock(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + _record_todo_write(registry, state, runtime_root, "Baseline") + assert _drain(registry, runtime_root).delivered == 1 + stale = _record_todo_write(registry, state, runtime_root, "Half-recorded", mark_committed=False) + # An unhooked writer edits the file after the crash: neither digest matches. + state.write_text( + state.read_text(encoding="utf-8") + "\n- [ ] (agent) foreign edit \n", + encoding="utf-8", + ) + + result = _drain(registry, runtime_root) + + assert result.ok is True + assert result.reseeded == 1 + assert result.no_op == 1 + assert [item["resolution"] for item in result.entries] == ["unproved", "seed"] + assert result.entries[0]["entry_id"] == stale.outcome.entry_id + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=10) + kinds = [tx["events"][0]["kind"] for tx in view["scan"]["transactions"]] + assert kinds == ["source_transaction_delivered", "source_transaction_unproved", "partition_seeded"] + listed = list_goal_todos(registry_path=registry, goal_id=GOAL_ID, runtime_root_arg=str(runtime_root)) + assert [todo["todo_id"] for todo in view["head"]["todos"]] == sorted( + todo["todo_id"] for todo in listed["todos"] + ) + assert len(view["head"]["todos"]) == 3 + assert view["head"]["partitions"]["todos"]["seq"] == 3 + seed_receipt = view["scan"]["transactions"][2]["receipts"][0] + assert seed_receipt["write_class"] == "reseed_after_crash_gap" + assert seed_receipt["source_bytes_digest"] == text_digest(state.read_text(encoding="utf-8")) + + +def test_lease_partition_entries_retain_complete_records_at_drain(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path) + directory = outbox.partition_directory(runtime_root, GOAL_ID, "leases") + record = { + "goal_id": GOAL_ID, + "todo_id": "todo_00000000000a", + "owner": "agent-a", + "idempotency_key": "k1", + "write_scopes": ["loopx/**"], + "version": 2, + "lease_epoch": 1, + "acquired_at": "2026-09-03T00:00:00+00:00", + "updated_at": "2026-09-03T00:01:00+00:00", + "expires_at": "2026-09-03T00:31:00+00:00", + "released_at": None, + "status": "active", + "lease_path": "/should/never/be/compared", + "acquire_ttl_seconds": 1800, + } + bytes_digest = text_digest(json.dumps(record, indent=2) + "\n") + entry_id = outbox.entry_identity(goal_id=GOAL_ID, partition="leases", seq=1, source_ref=bytes_digest) + outbox.durable_write_json( + directory / outbox.entry_file_name(1, entry_id, "prepared"), + { + "schema_version": outbox.OUTBOX_ENTRY_SCHEMA, + "goal_id": GOAL_ID, + "partition": "leases", + "seq": 1, + "entry_id": entry_id, + "writer": {"runtime": "typescript", "write_class": "task_lease_acquire", "operation_id": "op-1"}, + "source": { + "kind": "task_lease_record", + "previous_bytes_digest": None, + "bytes_digest": bytes_digest, + "lease": {"todo_id": record["todo_id"], "version": 2, "lease_epoch": 1, "status": "active", "updated_at": record["updated_at"]}, + "previous_lease": None, + "event_id": None, + }, + "source_root_digest": outbox.runtime_root_digest(runtime_root), + "projection": {"leases": [{"file_stem": record["todo_id"], "record": record}]}, + "partition_digest": None, + "prepared_at": "2026-09-03T00:01:00.000Z", + }, + ) + outbox.durable_write_json( + directory / outbox.entry_file_name(1, entry_id, "committed"), + {"schema_version": outbox.OUTBOX_COMMIT_SCHEMA, "entry_id": entry_id, "committed_at": "2026-09-03T00:01:00.100Z"}, + ) + + result = _drain(registry, runtime_root) + + assert result.ok is True + assert result.delivered == 1 + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + head = view["head"] + assert head["todos"] == [] + assert head["handoff_mode"] is None + assert head["leases"] == [record] + expected_digest = partition_digest({"leases": head["leases"]}) + assert head["partitions"]["leases"] == {"seq": 1, "partition_digest": expected_digest} + assert result.entries[0]["partition_digest"] == expected_digest + assert "/should/never/be/compared" in json.dumps(view) + + +def test_status_reports_backlog_candidate_and_growth_facts(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + empty = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert empty["ok"] is True + assert empty["config"]["status"] == "disabled" + assert empty["candidate"]["status"] == "missing" + assert empty["store_bytes"] == 0 + assert empty["retention_pressure"] is False + assert str(runtime_root) not in json.dumps(empty) + + _record_todo_write(registry, state, runtime_root, "Status fact") + pending = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert pending["outbox"]["todos"]["committed_pending"] == 1 + assert _drain(registry, runtime_root).delivered == 1 + + drained = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert drained["outbox"]["todos"] == { + "committed_pending": 0, + "prepared_only": 0, + "retired_residue": 0, + "next_seq": 0, + "cursor_last_seq": 1, + "cursor_last_entry_id": outbox.read_cursor(_todo_dir(runtime_root))["last_entry_id"], + "invalid": None, + } + assert drained["candidate"]["status"] == "loaded" + assert drained["candidate"]["cursor"] == "1" + assert drained["candidate"]["codec_agreement"] is True + assert drained["candidate"]["head_schema_version"] == "loopx_coordination_runtime_shadow_projection_v0" + assert drained["store_bytes"] > 0 + + +def test_capture_evidence_v1_reports_measured_facts_only(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Evidence fact") + drain = _drain(registry, runtime_root) + evidence = adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=drain) + assert adapter.valid_evidence_v1(evidence, goal_id=GOAL_ID) + assert evidence["outcome"] == "delivered" + assert evidence["entry"]["entry_id"] == capture.outcome.entry_id + assert evidence["source_transaction_correlated"] is True + assert evidence["durable_source_outbox"] is True + assert evidence["source_candidate_compared"] is False + assert evidence["parity_verdict"] == "not_evaluated" + assert evidence["drain"]["candidate_readback_verified"] is True + + deferred = adapter.DrainResult(goal_id=GOAL_ID, outcome="drain_deferred", reason_code="drain_lock_busy") + assert adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=deferred)["outcome"] == "drain_deferred" + pending = adapter.capture_evidence(goal_id=GOAL_ID, capture=capture.outcome, drain=None) + assert pending["outcome"] == "pending" + assert pending["drain"] is None + + for skipped_reason in ("partition_unchanged", "shadow_disabled"): + skipped = outbox.CaptureOutcome(partition="todos", skipped_reason=skipped_reason) + no_transaction = adapter.capture_evidence(goal_id=GOAL_ID, capture=skipped, drain=None) + assert no_transaction["outcome"] == "no_transaction" + assert no_transaction["reason_code"] == skipped_reason + # Nothing was recorded, so nothing durable or correlated may be claimed. + assert no_transaction["durable_source_outbox"] is False + assert no_transaction["source_transaction_correlated"] is False + assert adapter.valid_evidence_v1(no_transaction, goal_id=GOAL_ID) + failed = outbox.CaptureOutcome(partition="todos", failure={"reason_code": "outbox_prepare_failed", "error_class": "OSError"}) + failed_evidence = adapter.capture_evidence(goal_id=GOAL_ID, capture=failed, drain=None) + assert failed_evidence["outcome"] == "capture_failed" + assert failed_evidence["durable_source_outbox"] is False + assert failed_evidence["source_transaction_correlated"] is False + assert adapter.valid_evidence_v1(failed_evidence, goal_id=GOAL_ID) + + +def _commit_entry_calls(monkeypatch: pytest.MonkeyPatch) -> list[str]: + calls: list[str] = [] + real = adapter.effect_runtime_result + + def counting(method: str, params: object, **kwargs: object) -> object: + calls.append(method) + return real(method, params, **kwargs) + + monkeypatch.setattr(adapter, "effect_runtime_result", counting) + return calls + + +def test_crash_between_the_two_unlinks_leaves_residue_the_next_drain_reclaims( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Retired but half-removed") + real_remove = outbox.remove_entry_files + + def crash_between_unlinks(entry: outbox.OutboxEntry) -> None: + entry.prepared_path.unlink(missing_ok=True) + raise OSError("simulated crash between the prepared and committed unlinks") + + monkeypatch.setattr(outbox, "remove_entry_files", crash_between_unlinks) + first = _drain(registry, runtime_root) + assert first.outcome == "stopped" + assert first.reason_code == "shadow_drain_failed" + monkeypatch.setattr(outbox, "remove_entry_files", real_remove) + + # On disk: the cursor covers seq 1 and only the committed marker survives. + marker_name = outbox.entry_file_name(1, str(capture.outcome.entry_id), "committed") + names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) + assert names == sorted([marker_name, "drain-cursor.json"]) + assert outbox.read_cursor(_todo_dir(runtime_root))["last_seq"] == 1 + # The marker is retired residue, not corruption: listing stays valid. + assert outbox.list_entries(_todo_dir(runtime_root)) == [] + assert [path.name for path in outbox.retired_residue(_todo_dir(runtime_root))] == [marker_name] + summary = outbox.outbox_summary(runtime_root, GOAL_ID)["todos"] + assert summary["invalid"] is None + assert summary["retired_residue"] == 1 + assert summary["committed_pending"] == 0 + status = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert status["ok"] is True + assert status["outbox"]["todos"]["retired_residue"] == 1 + + calls = _commit_entry_calls(monkeypatch) + second = _drain(registry, runtime_root) + assert second.ok is True + assert second.outcome == "drained" + assert second.reclaimed_residue == 1 + assert (second.delivered, second.replayed) == (0, 0) + assert "coordination.runtime_shadow.commit_entry" not in calls + assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID, scan_limit=5) + assert view["cursor"] == "1" + + # A later write mints seq 2 from the cursor, never reusing the retired seq. + later = _record_todo_write(registry, state, runtime_root, "After the reclaim") + assert later.outcome.seq == 2 + third = _drain(registry, runtime_root) + assert third.delivered == 1 + assert third.reclaimed_residue == 0 + + +def test_crash_after_the_cursor_but_before_any_unlink_is_reclaimed_without_a_store_call( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Cursor written, files untouched") + real_remove = outbox.remove_entry_files + + def crash_before_unlinks(entry: outbox.OutboxEntry) -> None: + raise OSError("simulated crash after the cursor write") + + monkeypatch.setattr(outbox, "remove_entry_files", crash_before_unlinks) + assert _drain(registry, runtime_root).outcome == "stopped" + monkeypatch.setattr(outbox, "remove_entry_files", real_remove) + names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) + entry_id = str(capture.outcome.entry_id) + assert names == [ + outbox.entry_file_name(1, entry_id, "committed"), + outbox.entry_file_name(1, entry_id, "prepared"), + "drain-cursor.json", + ] + assert outbox.list_entries(_todo_dir(runtime_root)) == [] + + calls = _commit_entry_calls(monkeypatch) + result = _drain(registry, runtime_root) + assert result.ok is True + assert result.reclaimed_residue == 2 + assert "coordination.runtime_shadow.commit_entry" not in calls + assert list(_todo_dir(runtime_root).iterdir()) == [_todo_dir(runtime_root) / "drain-cursor.json"] + + +def test_an_orphan_marker_above_the_cursor_is_still_corruption(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Marker without its prepared file") + (_todo_dir(runtime_root) / outbox.entry_file_name(1, str(capture.outcome.entry_id), "prepared")).unlink() + with pytest.raises(outbox.OutboxError) as raised: + outbox.list_entries(_todo_dir(runtime_root)) + assert raised.value.reason_code == "outbox_file_invalid" + result = _drain(registry, runtime_root) + assert result.outcome == "stopped" + assert result.reason_code == "outbox_file_invalid" + status = adapter.local_authority_shadow_status(registry_path=registry, runtime_root=runtime_root, goal_id=GOAL_ID) + assert status["ok"] is False + assert status["outbox"]["todos"]["invalid"] == "outbox_file_invalid" + + +def test_drain_fails_closed_on_an_entry_recorded_for_another_runtime_root(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _record_todo_write(registry, state, runtime_root, "Written under a foreign root") + entry_id = str(capture.outcome.entry_id) + prepared_path = _todo_dir(runtime_root) / outbox.entry_file_name(1, entry_id, "prepared") + record = json.loads(prepared_path.read_text(encoding="utf-8")) + record["source_root_digest"] = outbox.runtime_root_digest(tmp_path / "elsewhere") + outbox.durable_write_json(prepared_path, record) + + result = _drain(registry, runtime_root) + + assert result.outcome == "stopped" + assert result.reason_code == "source_root_mismatch" + assert result.delivered == 0 + assert result.pending_after == 1 + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1] diff --git a/tests/control_plane/test_local_authority_shadow_outbox.py b/tests/control_plane/test_local_authority_shadow_outbox.py new file mode 100644 index 0000000000..907dc8b140 --- /dev/null +++ b/tests/control_plane/test_local_authority_shadow_outbox.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter +from loopx.control_plane.coordination import local_authority_shadow_outbox as outbox +from loopx.control_plane.coordination.local_authority_shadow_projection import ( + ProjectionValueError, + canonical_bytes, + lease_partition_projection, + partition_digest, + sha256_digest, + text_digest, + todo_partition_projection, +) +from loopx.file_lock import exclusive_file_lock +from loopx.history import load_registry +from loopx.registry import find_registry_goal +from loopx.todos import add_goal_todo + + +GOAL_ID = "goal-outbox" + + +def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-03T00:00:00+00:00\n" + "---\n\n" + "## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": { + "agent_model": "peer_v1", + "registered_agents": ["agent-a"], + }, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def _capture( + registry: Path, + state: Path, + runtime_root: Path, + *, + original_text: str, + enabled: bool = True, + write_class: str = "todo_add", +) -> outbox.TodoPartitionCapture: + goal = find_registry_goal(load_registry(registry), GOAL_ID) + return outbox.TodoPartitionCapture.begin( + enabled=enabled, + runtime_root=runtime_root, + goal_id=GOAL_ID, + state_path=state, + write_class=write_class, + original_text=original_text, + projector=adapter.todo_partition_projector(goal, state_path=state), + ) + + +def _add_todo(registry: Path, text: str) -> str: + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text=text, + task_class="advancement_task", + ) + assert result["ok"] is True + return str(result["todo_id"]) + + +def _todo_dir(runtime_root: Path) -> Path: + return outbox.partition_directory(runtime_root, GOAL_ID, "todos") + + +def test_capture_records_prepared_then_committed_and_skips_prose_only_writes(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + original = state.read_text(encoding="utf-8") + todo_id = _add_todo(registry, "Bind the shadow to the primary transaction.") + new_text = state.read_text(encoding="utf-8") + + capture = _capture(registry, state, runtime_root, original_text=original) + capture.prepare(new_text) + names = sorted(path.name for path in _todo_dir(runtime_root).iterdir()) + assert len(names) == 1 + assert names[0].endswith(".prepared.json") + assert capture.outcome.entry_id is not None + assert capture.outcome.seq == 1 + assert capture.outcome.source_bytes_digest == text_digest(new_text) + assert capture.outcome.entry_id == outbox.entry_identity( + goal_id=GOAL_ID, partition="todos", seq=1, source_ref=text_digest(new_text) + ) + + capture.committed() + entries = outbox.list_entries(_todo_dir(runtime_root)) + assert [entry.is_committed for entry in entries] == [True] + entry = entries[0] + assert entry.prepared["schema_version"] == outbox.OUTBOX_ENTRY_SCHEMA + assert entry.committed is not None + assert entry.committed["schema_version"] == outbox.OUTBOX_COMMIT_SCHEMA + assert entry.prepared["source"] == { + "kind": "markdown_active_state", + "previous_bytes_digest": text_digest(original), + "bytes_digest": text_digest(new_text), + "lease": None, + "event_id": None, + } + assert entry.prepared["writer"] == { + "runtime": "python", + "write_class": "todo_add", + "operation_id": None, + } + projection = entry.projection() + assert projection is not None + assert projection["handoff_mode"] == "hard_lease" + assert [item["todo_id"] for item in projection["todos"]] == [todo_id] + assert entry.recorded_partition_digest() == partition_digest(projection) + assert str(runtime_root) not in json.dumps(entry.prepared) + + prose_only = new_text + "\nOperator note that changes no coordination fact.\n" + prose = _capture(registry, state, runtime_root, original_text=new_text, write_class="todo_update") + prose.prepare(prose_only) + prose.committed() + assert prose.outcome.entry_id is None + assert prose.outcome.skipped_reason == "partition_unchanged" + assert len(outbox.list_entries(_todo_dir(runtime_root))) == 1 + + _add_todo(registry, "Second coordination fact.") + third = _capture(registry, state, runtime_root, original_text=new_text) + third.prepare(state.read_text(encoding="utf-8")) + third.committed() + assert third.outcome.seq == 2 + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1, 2] + + +def test_disabled_capture_creates_nothing(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + capture = _capture(registry, state, runtime_root, original_text="", enabled=False) + capture.prepare("# anything") + capture.committed() + assert capture.outcome.skipped_reason == "shadow_disabled" + assert capture.outcome.failure is None + assert not (runtime_root / "authority-shadow").exists() + + +def test_event_branch_records_projection_in_committed_marker(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + empty = state.read_text(encoding="utf-8") + baseline_id = _add_todo(registry, "Baseline coordination fact.") + original = state.read_text(encoding="utf-8") + baseline = _capture(registry, state, runtime_root, original_text=empty) + baseline.prepare(original) + baseline.committed() + assert baseline.outcome.seq == 1 + + # An appended event that changes no compared field retires its own entry. + unchanged = _capture(registry, state, runtime_root, original_text=original, write_class="todo_complete_event_projection") + unchanged.prepare(original, event_id="evt-noop") + assert unchanged.outcome.entry_id is not None + unchanged.committed(projection_from_disk=True) + assert unchanged.outcome.entry_id is None + assert unchanged.outcome.skipped_reason == "partition_unchanged" + assert [entry.seq for entry in outbox.list_entries(_todo_dir(runtime_root))] == [1] + + capture = _capture(registry, state, runtime_root, original_text=original, write_class="todo_complete_event_projection") + capture.prepare(original, event_id="evt-1") + [_baseline, entry] = outbox.list_entries(_todo_dir(runtime_root)) + assert entry.prepared["source"]["kind"] == "state_event_log" + assert entry.prepared["source"]["event_id"] == "evt-1" + assert entry.prepared["projection"] is None + assert entry.prepared["writer"]["operation_id"] == "evt-1" + assert entry.source_ref == "event:evt-1" + assert not entry.is_committed + + todo_id = _add_todo(registry, "Landed by the event append.") + capture.committed(projection_from_disk=True) + [_baseline, entry] = outbox.list_entries(_todo_dir(runtime_root)) + assert entry.is_committed + projection = entry.projection() + assert projection is not None + assert sorted(item["todo_id"] for item in projection["todos"]) == sorted([baseline_id, todo_id]) + assert entry.recorded_partition_digest() == partition_digest(projection) + assert capture.outcome.partition_digest == partition_digest(projection) + + +def test_prepared_only_entries_resolve_from_source_probes(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + original = state.read_text(encoding="utf-8") + _add_todo(registry, "Crash between write and marker.") + new_text = state.read_text(encoding="utf-8") + capture = _capture(registry, state, runtime_root, original_text=original) + capture.prepare(new_text) + [entry] = outbox.list_entries(_todo_dir(runtime_root)) + assert not entry.is_committed + + def resolve(text: str) -> str: + return outbox.resolve_prepared_only_entry( + entry, + markdown_text_reader=lambda: text, + lease_record_reader=None, + event_presence_reader=None, + ) + + assert resolve(new_text) == "committed" + assert resolve(original) == "abandoned" + assert resolve(new_text + "\n- [ ] (agent) foreign edit\n") == "unproved" + + event_entry = outbox.OutboxEntry( + partition="todos", + seq=2, + entry_id="local-shadow-tx-" + "0" * 64, + prepared_path=tmp_path / "unused.prepared.json", + committed_path=None, + prepared={"source": {"kind": "state_event_log", "event_id": "evt-9"}}, + committed=None, + ) + assert outbox.resolve_prepared_only_entry( + event_entry, markdown_text_reader=None, lease_record_reader=None, + event_presence_reader=lambda event_id: event_id == "evt-9", + ) == "unproved" + assert outbox.resolve_prepared_only_entry( + event_entry, markdown_text_reader=None, lease_record_reader=None, + event_presence_reader=lambda _event_id: False, + ) == "abandoned" + + planned = {"todo_id": "todo-a", "version": 2, "lease_epoch": 1, "status": "active", "updated_at": "t2"} + previous = {"todo_id": "todo-a", "version": 1, "lease_epoch": 1, "status": "active", "updated_at": "t1"} + lease_entry = outbox.OutboxEntry( + partition="leases", + seq=1, + entry_id="local-shadow-tx-" + "1" * 64, + prepared_path=tmp_path / "unused.prepared.json", + committed_path=None, + prepared={"source": {"kind": "task_lease_record", "lease": planned, "previous_lease": previous}}, + committed=None, + ) + + def lease_resolve(current: dict[str, object] | None) -> str: + return outbox.resolve_prepared_only_entry( + lease_entry, + markdown_text_reader=None, + lease_record_reader=lambda _todo_id: current, + event_presence_reader=None, + ) + + assert lease_resolve({**planned, "owner": "agent-a"}) == "committed" + assert lease_resolve({**previous, "owner": "agent-a"}) == "abandoned" + assert lease_resolve({**planned, "version": 9}) == "unproved" + assert lease_resolve(None) == "unproved" + + +def test_sequence_advances_past_the_drain_cursor_and_lists_oldest_first(tmp_path: Path) -> None: + _registry, _state, runtime_root = _fixture(tmp_path) + directory = _todo_dir(runtime_root) + directory.mkdir(parents=True) + outbox.write_cursor( + directory, + partition="todos", + last_seq=5, + last_entry_id="local-shadow-tx-" + "a" * 64, + last_partition_digest=None, + last_cursor="5", + last_provider_revision="rev-5", + ) + assert outbox.next_seq(directory) == 6 + seed = outbox.SeedSource(partition="todos", projection={"handoff_mode": "hard_lease", "todos": []}) + first = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + second = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + assert (first.seq, second.seq) == (6, 7) + assert [entry.seq for entry in outbox.list_entries(directory)] == [6, 7] + assert all(entry.is_committed for entry in outbox.list_entries(directory)) + assert outbox.latest_partition_digest(directory) == partition_digest(seed.projection) + summary = outbox.outbox_summary(runtime_root, GOAL_ID) + assert summary["todos"]["committed_pending"] == 2 + assert summary["todos"]["cursor_last_seq"] == 5 + assert summary["leases"] == { + "committed_pending": 0, + "prepared_only": 0, + "retired_residue": 0, + "next_seq": 0, + "cursor_last_seq": None, + "cursor_last_entry_id": None, + "invalid": None, + } + outbox.remove_entry_files(first) + assert [entry.seq for entry in outbox.list_entries(directory)] == [7] + + +def test_canonical_projection_rejects_floats_and_bad_lease_identity() -> None: + with pytest.raises(ProjectionValueError): + canonical_bytes({"version": 1.0}) + with pytest.raises(ProjectionValueError): + todo_partition_projection(handoff_mode="hard_lease", todos=[{"todo_id": "a", "status": 2.5}]) + assert canonical_bytes({"b": 1, "a": [True, None, "\u00e9"]}) == '{"a":[true,null,"\u00e9"],"b":1}'.encode("utf-8") + assert sha256_digest({"b": [1, None], "a": "x"}) == sha256_digest({"a": "x", "b": [1, None]}) + with pytest.raises(ProjectionValueError): + lease_partition_projection([("todo-a", {"goal_id": "other", "todo_id": "todo-a"})], goal_id=GOAL_ID) + projection = lease_partition_projection( + [("todo-b", {"goal_id": GOAL_ID, "todo_id": "todo-b", "version": 1, "extra": "retained"}), + ("todo-a", {"goal_id": GOAL_ID, "todo_id": "todo-a", "version": 2, "status": "active"})], + goal_id=GOAL_ID, + ) + assert [lease["todo_id"] for lease in projection["leases"]] == ["todo-a", "todo-b"] + assert projection["leases"][1]["extra"] == "retained" + + +def test_capture_failure_is_typed_and_never_raises(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + blocker = runtime_root / "authority-shadow" + blocker.parent.mkdir(parents=True) + blocker.write_text("not a directory", encoding="utf-8") + capture = _capture(registry, state, runtime_root, original_text="") + capture.prepare("---\ngoal_id: goal-outbox\n---\n\n## Agent Todo\n\n- [ ] (agent) x \n") + capture.committed() + assert capture.outcome.entry_id is None + assert capture.outcome.failure is not None + assert capture.outcome.failure["reason_code"] == "outbox_prepare_failed" + + +def test_primary_lock_probe_reports_held_locks(tmp_path: Path) -> None: + target = tmp_path / "ACTIVE_GOAL_STATE.md" + target.write_text("", encoding="utf-8") + assert adapter.primary_lock_is_free(target) is True + with exclusive_file_lock(target, timeout_seconds=1.0, operation="test_hold"): + assert adapter.primary_lock_is_free(target) is False + assert adapter.primary_lock_is_free(target) is True + + +def test_prepared_records_must_bind_their_directory_identity_and_source(tmp_path: Path) -> None: + registry, state, runtime_root = _fixture(tmp_path) + original = state.read_text(encoding="utf-8") + _add_todo(registry, "Bound to this goal and partition.") + capture = _capture(registry, state, runtime_root, original_text=original) + capture.prepare(state.read_text(encoding="utf-8")) + capture.committed() + directory = _todo_dir(runtime_root) + [entry] = outbox.list_entries(directory) + assert entry.prepared["goal_id"] == GOAL_ID + assert entry.prepared["partition"] == "todos" + assert entry.prepared["source_root_digest"] == outbox.runtime_root_digest(runtime_root) + assert outbox.record_source_ref(entry.prepared) == text_digest(state.read_text(encoding="utf-8")) + + def tampered(**changes: object) -> None: + record = dict(entry.prepared) + for key, value in changes.items(): + if isinstance(value, dict) and isinstance(record.get(key), dict): + record[key] = {**record[key], **value} + else: + record[key] = value + outbox.durable_write_json(entry.prepared_path, record) + with pytest.raises(outbox.OutboxError) as raised: + outbox.list_entries(directory) + assert raised.value.reason_code == "outbox_file_invalid" + + tampered(goal_id="goal-other") + tampered(partition="leases") + tampered(source={"bytes_digest": text_digest("some other bytes")}) + tampered(source_root_digest="not-a-digest") + tampered(writer={"runtime": "ruby"}) + tampered(source={"kind": "unknown_source"}) + outbox.durable_write_json(entry.prepared_path, entry.prepared) + assert len(outbox.list_entries(directory)) == 1 + + # Seed entries bind their identity through the partition digest instead. + lease_seed = outbox.write_seed_entry( + runtime_root=runtime_root, + goal_id=GOAL_ID, + seed=outbox.lease_seed_source(runtime_root, GOAL_ID), + ) + assert outbox.record_source_ref(lease_seed.prepared) == f"seed:{lease_seed.recorded_partition_digest()}" + lease_directory = outbox.partition_directory(runtime_root, GOAL_ID, "leases") + assert [item.entry_id for item in outbox.list_entries(lease_directory)] == [lease_seed.entry_id] + + +def test_retired_residue_is_defined_by_the_cursor_watermark(tmp_path: Path) -> None: + _registry, _state, runtime_root = _fixture(tmp_path) + directory = _todo_dir(runtime_root) + seed = outbox.SeedSource(partition="todos", projection={"handoff_mode": "hard_lease", "todos": []}) + first = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + second = outbox.write_seed_entry(runtime_root=runtime_root, goal_id=GOAL_ID, seed=seed) + assert outbox.retired_residue(directory) == [] + outbox.write_cursor( + directory, + partition="todos", + last_seq=first.seq, + last_entry_id=first.entry_id, + last_partition_digest=first.recorded_partition_digest(), + last_cursor="1", + last_provider_revision="rev-1", + ) + assert [path.name for path in outbox.retired_residue(directory)] == sorted( + [first.committed_path.name, first.prepared_path.name] # type: ignore[union-attr] + ) + assert [entry.seq for entry in outbox.list_entries(directory)] == [second.seq] + assert outbox.next_seq(directory) == 3 + assert outbox.reclaim_retired_residue(directory) == 2 + assert outbox.retired_residue(directory) == [] + assert [entry.seq for entry in outbox.list_entries(directory)] == [second.seq] diff --git a/tests/control_plane/test_runtime_shadow_writer_capture.py b/tests/control_plane/test_runtime_shadow_writer_capture.py new file mode 100644 index 0000000000..a37a4919af --- /dev/null +++ b/tests/control_plane/test_runtime_shadow_writer_capture.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from loopx.control_plane.coordination import local_authority_shadow_adapter as adapter +from loopx.control_plane.work_items.task_lease import ( + acquire_task_lease, + release_task_lease, + renew_task_lease, + transfer_task_lease, +) +from loopx.todos import add_goal_todo, update_goal_todo + + +GOAL_ID = "runtime-shadow-writer" + + +def _fixture(tmp_path: Path, *, enabled: bool) -> tuple[Path, Path, Path]: + repo = tmp_path / "repo" + repo.mkdir() + state = repo / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\n" + f"goal_id: {GOAL_ID}\n" + "handoff_mode: hard_lease\n" + "updated_at: 2026-09-04T00:00:00+00:00\n" + "---\n\n## Agent Todo\n\n", + encoding="utf-8", + ) + runtime_root = tmp_path / "runtime" + coordination: dict[str, object] = { + "agent_model": "peer_v1", + "registered_agents": ["agent-a", "agent-b"], + } + if enabled: + coordination["runtime_shadow"] = { + "enabled": True, + "schema_version": "loopx_coordination_runtime_shadow_config_v0", + "provider": "file_v0", + } + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": GOAL_ID, + "domain": "harness_self_improvement", + "status": "active", + "repo": str(repo), + "state_file": state.name, + "adapter": {"kind": "harness_self_improvement"}, + "coordination": coordination, + } + ], + } + ), + encoding="utf-8", + ) + return registry, state, runtime_root + + +def test_runtime_shadow_todo_writer_captures_full_records_and_reuses_one_store( + tmp_path: Path, +) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + + added = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Retain complete canonical Todo fields.", + task_class="advancement_task", + action_kind="implement", + claimed_by="agent-a", + ) + updated = update_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=str(added["todo_id"]), + note="transaction-bound", + agent_id="agent-a", + ) + + assert added["coordination_runtime_shadow"]["outcome"] == "delivered" + assert updated["coordination_runtime_shadow"]["outcome"] == "delivered" + view = adapter.read_local_authority_shadow( + runtime_root=runtime_root, + goal_id=GOAL_ID, + scan_limit=10, + ) + head = view["head"] + assert head["schema_version"] == "loopx_coordination_runtime_shadow_projection_v0" + assert head["todos"][0]["text"] == "Retain complete canonical Todo fields." + assert head["todos"][0]["note"] == "transaction-bound" + assert head["todo_read_model"]["todo_count"] == 1 + assert view["cursor"] == "2" + assert not (runtime_root / "authority-shadow" / "file" / GOAL_ID).exists() + + +def test_runtime_shadow_todo_writer_is_zero_effect_by_default(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=False) + + result = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Default-off capture.", + task_class="advancement_task", + ) + + assert result["coordination_runtime_shadow"]["outcome"] == "no_transaction" + assert result["coordination_runtime_shadow"]["reason_code"] == "shadow_disabled" + assert not (runtime_root / "authority-shadow").exists() + + +def test_runtime_shadow_native_lease_writers_capture_complete_records(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=True) + added = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Capture the native lease transaction.", + task_class="advancement_task", + ) + todo_id = str(added["todo_id"]) + + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-a", + idempotency_key="runtime-shadow-lease", + write_scopes=["loopx/**"], + ttl_seconds=120, + ) + renewed = renew_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-a", + idempotency_key="runtime-shadow-lease", + ttl_seconds=180, + expected_version=int(acquired["lease"]["version"]), + ) + transferred = transfer_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-a", + idempotency_key="runtime-shadow-lease", + new_owner="agent-b", + new_idempotency_key="runtime-shadow-lease-b", + ttl_seconds=180, + expected_version=int(renewed["lease"]["version"]), + ) + released = release_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=todo_id, + owner="agent-b", + idempotency_key="runtime-shadow-lease-b", + expected_version=int(transferred["lease"]["version"]), + ) + + assert acquired["coordination_runtime_shadow"]["outcome"] == "delivered" + assert renewed["coordination_runtime_shadow"]["outcome"] == "delivered" + assert transferred["coordination_runtime_shadow"]["outcome"] == "delivered" + assert released["coordination_runtime_shadow"]["outcome"] == "delivered" + view = adapter.read_local_authority_shadow(runtime_root=runtime_root, goal_id=GOAL_ID) + lease = view["head"]["leases"][0] + assert lease["goal_id"] == GOAL_ID + assert lease["owner"] == "agent-b" + assert lease["idempotency_key"] == "runtime-shadow-lease-b" + assert lease["write_scopes"] == ["loopx/**"] + assert lease["status"] == "released" + + +def test_runtime_shadow_native_lease_writer_is_zero_effect_by_default(tmp_path: Path) -> None: + registry, _state, runtime_root = _fixture(tmp_path, enabled=False) + added = add_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + role="agent", + text="Do not capture an unconfigured lease.", + task_class="advancement_task", + claimed_by="agent-a", + ) + + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id=GOAL_ID, + todo_id=str(added["todo_id"]), + owner="agent-a", + idempotency_key="default-off-lease", + write_scopes=["loopx/**"], + ttl_seconds=120, + ) + + assert "coordination_runtime_shadow_capture" not in acquired + assert "coordination_runtime_shadow" not in acquired + assert not (runtime_root / "authority-shadow").exists() diff --git a/tests/control_plane_ts/local_authority_shadow_outbox.test.ts b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts new file mode 100644 index 0000000000..b8358971a7 --- /dev/null +++ b/tests/control_plane_ts/local_authority_shadow_outbox.test.ts @@ -0,0 +1,368 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { FileAuthorityStore } from "../../loopx/control_plane/coordination/file_authority_store.ts"; +import { + LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA, + LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1, + LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA, + LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA, + commitLocalAuthorityShadowEntry, + composeLocalAuthorityShadowHead, + localAuthorityShadowHeadDigest, + readLocalAuthorityShadow, +} from "../../loopx/control_plane/coordination/local_authority_shadow.ts"; +import { + LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA, + LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA, + beginLeaseOutboxEntry, + decodeLocalAuthorityShadowBinding, + leaseRecordDigest, + outboxEntryIdentity, + sha256Digest, +} from "../../loopx/control_plane/coordination/local_authority_shadow_outbox.ts"; + +const execFileAsync = promisify(execFile); +const GOAL = "goal-a"; +const HEX = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +function entryId(seq: number, sourceRef: string): string { + return outboxEntryIdentity(GOAL, "todos", seq, sourceRef); +} + +function todoEntry(seq: number, digest: string, resolution = "committed") { + return { + entry_id: entryId(seq, `sha256:${HEX}`), + partition: "todos", + seq, + writer: { runtime: "python", write_class: "todo_add", operation_id: null }, + source: { + kind: "markdown_active_state", + previous_bytes_digest: null, + bytes_digest: `sha256:${HEX}`, + lease: null, + event_id: null, + }, + source_root_digest: `sha256:${HEX}`, + prepared_at: "2026-09-03T00:00:00.000Z", + committed_at: "2026-09-03T00:00:00.100Z", + resolution, + }; +} + +function commitRequest(root: string, seq: number, todos: object[], resolution = "committed") { + const projection = { handoff_mode: "hard_lease", todos }; + return { + schema_version: "loopx_coordination_runtime_shadow_commit_entry_request_v0", + runtime_root: root, + goal_id: GOAL, + entry: todoEntry(seq, `sha256:${HEX}`, resolution), + partition_projection: projection, + partition_digest: `sha256:${"b".repeat(64)}`, + }; +} + +function noOpRequest(root: string, seq: number, resolution: "abandoned" | "unproved") { + return { + schema_version: "loopx_coordination_runtime_shadow_commit_entry_request_v0", + runtime_root: root, + goal_id: GOAL, + entry: todoEntry(seq, `sha256:${HEX}`, resolution), + partition_projection: null, + partition_digest: null, + }; +} + +async function tempRoot(t: test.TestContext): Promise { + const root = await mkdtemp(join(tmpdir(), "loopx-shadow-outbox-")); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +test("commit_entry folds one partition into a v1 head and binds the receipt to the entry", async (t) => { + const root = await tempRoot(t); + const todos = [{ todo_id: "todo-a", status: "open" }]; + + const result = await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + + assert.equal(result.schema_version, LOCAL_AUTHORITY_SHADOW_COMMIT_ENTRY_RESULT_SCHEMA); + assert.equal(result.outcome, "delivered"); + assert.equal(result.no_op, false); + assert.equal(result.cursor, "1"); + const store = new FileAuthorityStore(join(root, "authority-shadow", "file-v0"), GOAL); + const loaded = await store.loadAuthority(); + assert.equal(loaded.status, "loaded"); + if (loaded.status !== "loaded") return; + assert.equal(loaded.head.schema_version, LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1); + assert.equal(loaded.head.handoff_mode, "hard_lease"); + assert.deepEqual(loaded.head.todos, todos); + assert.deepEqual(loaded.head.leases, []); + assert.deepEqual(loaded.head.partitions, { + todos: { seq: 1, partition_digest: `sha256:${"b".repeat(64)}` }, + leases: null, + }); + assert.equal(result.head_digest, localAuthorityShadowHeadDigest(loaded.head)); + const receipt = await store.readReceipt(entryId(1, `sha256:${HEX}`)); + assert.equal(receipt.status, "found"); + if (receipt.status !== "found") return; + const record = receipt.receipts[0] as Record; + assert.equal(record.schema_version, LOCAL_AUTHORITY_SHADOW_TRANSACTION_RECEIPT_SCHEMA); + assert.equal(record.entry_id, entryId(1, `sha256:${HEX}`)); + assert.equal(record.source_transaction_correlated, true); + assert.equal(record.durable_source_outbox, true); + assert.equal(record.parity_verdict, "not_evaluated"); + assert.equal(record.primary_authority, "legacy_local"); + assert.equal(record.provider_to_local_writes, false); + assert.equal(record.candidate_read_for_decision, false); +}); + +test("commit_entry replays only when the existing receipt carries the same partition digest", async (t) => { + const root = await tempRoot(t); + const todos = [{ todo_id: "todo-a", status: "open" }]; + assert.equal((await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos))).outcome, "delivered"); + + const replay = await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + assert.equal(replay.outcome, "replayed"); + assert.equal(replay.cursor, "1"); + + const tampered = commitRequest(root, 1, todos); + tampered.partition_digest = `sha256:${"c".repeat(64)}`; + const mismatch = await commitLocalAuthorityShadowEntry(tampered); + assert.equal(mismatch.outcome, "protocol_mismatch"); + assert.equal(mismatch.reason_code, "transaction_receipt_mismatch"); + + const page = await new FileAuthorityStore(join(root, "authority-shadow", "file-v0"), GOAL) + .scanCommitted(null, 10); + assert.equal(page.status, "page"); + if (page.status === "page") assert.equal(page.transactions.length, 1); +}); + +test("no-op resolutions keep the sequence auditable without touching compared fields", async (t) => { + const root = await tempRoot(t); + const todos = [{ todo_id: "todo-a", status: "open" }]; + const first = await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + + const abandoned = await commitLocalAuthorityShadowEntry(noOpRequest(root, 2, "abandoned")); + const unproved = await commitLocalAuthorityShadowEntry(noOpRequest(root, 3, "unproved")); + + assert.equal(abandoned.outcome, "delivered"); + assert.equal(abandoned.no_op, true); + assert.equal(unproved.no_op, true); + assert.equal(unproved.cursor, "3"); + assert.equal(abandoned.head_digest, first.head_digest); + const store = new FileAuthorityStore(join(root, "authority-shadow", "file-v0"), GOAL); + const page = await store.scanCommitted(null, 10); + assert.equal(page.status, "page"); + if (page.status !== "page") return; + assert.deepEqual( + page.transactions.map((transaction) => (transaction.events[0] as Record).kind), + ["source_transaction_delivered", "source_transaction_abandoned", "source_transaction_unproved"], + ); + const loaded = await store.loadAuthority(); + if (loaded.status === "loaded") { + assert.deepEqual((loaded.head.partitions as Record).todos, { + seq: 1, + partition_digest: `sha256:${"b".repeat(64)}`, + }); + } +}); + +test("commit_entry rejects projection/resolution combinations that would misstate a transaction", async (t) => { + const root = await tempRoot(t); + const withProjection = noOpRequest(root, 1, "abandoned") as Record; + withProjection.partition_projection = { handoff_mode: "hard_lease", todos: [] }; + withProjection.partition_digest = `sha256:${"b".repeat(64)}`; + await assert.rejects(commitLocalAuthorityShadowEntry(withProjection), /must not carry/u); + + const withoutProjection = commitRequest(root, 1, []) as Record; + withoutProjection.partition_projection = null; + withoutProjection.partition_digest = null; + await assert.rejects(commitLocalAuthorityShadowEntry(withoutProjection), /requires partition_projection/u); + + const badId = commitRequest(root, 1, []); + badId.entry.entry_id = "local-shadow:abc"; + await assert.rejects(commitLocalAuthorityShadowEntry(badId), /entry\.entry_id/u); + + const extra = { ...commitRequest(root, 1, []), observation_id: "x" }; + await assert.rejects(commitLocalAuthorityShadowEntry(extra), /unsupported fields/u); +}); + +test("a v0 observation head is accepted as the starting point for partition folds", () => { + const v0 = { + schema_version: "loopx_local_authority_shadow_projection_v0", + goal_id: GOAL, + handoff_mode: "hard_lease", + todos: [{ todo_id: "todo-a", status: "open" }], + leases: [{ todo_id: "todo-a", version: 1 }], + }; + const folded = composeLocalAuthorityShadowHead( + v0, + GOAL, + { partition: "leases", seq: 4 }, + { leases: [] }, + `sha256:${"d".repeat(64)}`, + ); + assert.equal(folded.schema_version, LOCAL_AUTHORITY_SHADOW_PROJECTION_SCHEMA_V1); + assert.equal(folded.handoff_mode, "hard_lease"); + assert.deepEqual(folded.todos, v0.todos); + assert.deepEqual(folded.leases, []); + assert.deepEqual(folded.partitions, { + todos: null, + leases: { seq: 4, partition_digest: `sha256:${"d".repeat(64)}` }, + }); +}); + +test("read returns head, comparison digest, and a bounded scan page", async (t) => { + const root = await tempRoot(t); + const missing = await readLocalAuthorityShadow({ + schema_version: "loopx_coordination_runtime_shadow_outbox_read_v0", + runtime_root: root, + goal_id: GOAL, + scan_after_cursor: null, + scan_limit: 10, + }); + assert.equal(missing.schema_version, LOCAL_AUTHORITY_SHADOW_READ_RESULT_SCHEMA); + assert.equal(missing.status, "missing"); + assert.equal(missing.head, null); + + const todos = [{ todo_id: "todo-a", status: "open" }]; + await commitLocalAuthorityShadowEntry(commitRequest(root, 1, todos)); + await commitLocalAuthorityShadowEntry(noOpRequest(root, 2, "abandoned")); + const view = await readLocalAuthorityShadow({ + schema_version: "loopx_coordination_runtime_shadow_outbox_read_v0", + runtime_root: root, + goal_id: GOAL, + scan_after_cursor: null, + scan_limit: 1, + }); + assert.equal(view.status, "loaded"); + assert.equal(view.cursor, "2"); + assert.match(String(view.store_identity), /^file:[0-9a-f]{32}$/u); + assert.equal(view.head_digest, localAuthorityShadowHeadDigest(view.head as Record)); + const scan = view.scan as { transactions: Record[]; next_cursor: string | null; has_more: boolean }; + assert.equal(scan.transactions.length, 1); + assert.equal(scan.has_more, true); + assert.equal(scan.transactions[0].operation_id, entryId(1, `sha256:${HEX}`)); + assert.equal(scan.transactions[0].projection_digest, view.head_digest); + assert.equal("projection" in scan.transactions[0], false); +}); + +test("lease outbox entries are two-phase, durable, and skipped without a binding", async (t) => { + const root = await tempRoot(t); + const leaseDirectory = join(root, "goals", GOAL, "task-leases"); + const other = { goal_id: GOAL, todo_id: "todo-b", version: 1, status: "active" }; + await mkdir(leaseDirectory, { recursive: true }); + await writeFile(join(leaseDirectory, "todo-b.json"), `${JSON.stringify(other, null, 2)}\n`); + const planned = { goal_id: GOAL, todo_id: "todo-a", version: 2, lease_epoch: 1, status: "active", updated_at: "t2" }; + + assert.equal(decodeLocalAuthorityShadowBinding(undefined), null); + assert.equal(decodeLocalAuthorityShadowBinding({ provider: "file_v0" }), null); + assert.deepEqual( + decodeLocalAuthorityShadowBinding({ + schema_version: "loopx_coordination_runtime_shadow_binding_v0", + provider: "file_v0", + }), + { schema_version: "loopx_coordination_runtime_shadow_binding_v0", provider: "file_v0" }, + ); + + const capture = await beginLeaseOutboxEntry({ + runtime_root: root, + goal_id: GOAL, + lease_directory: leaseDirectory, + write_class: "task_lease_acquire", + operation_id: "op-1", + previous_lease: null, + planned_lease: planned, + }); + assert.equal(capture.failure, null); + assert.equal(capture.seq, 1); + assert.equal(capture.source_bytes_digest, leaseRecordDigest(planned)); + assert.equal(capture.entry_id, outboxEntryIdentity(GOAL, "leases", 1, leaseRecordDigest(planned))); + const directory = join(root, "authority-shadow", "outbox", GOAL, "leases"); + let names = await readdir(directory); + assert.deepEqual(names, [`0000000001-${capture.entry_id}.prepared.json`]); + const prepared = JSON.parse(await readFile(join(directory, names[0]), "utf8")); + assert.equal(prepared.schema_version, LOCAL_AUTHORITY_SHADOW_OUTBOX_ENTRY_SCHEMA); + assert.equal(prepared.partition, "leases"); + assert.equal(prepared.partition_digest, null); + assert.equal(prepared.source_root_digest, sha256Digest(resolve(root))); + assert.deepEqual(prepared.source.lease, { + todo_id: "todo-a", + version: 2, + lease_epoch: 1, + status: "active", + updated_at: "t2", + }); + assert.deepEqual( + prepared.projection.leases.map((item: { file_stem: string }) => item.file_stem), + ["todo-a", "todo-b"], + ); + assert.deepEqual(prepared.projection.leases[0].record, planned); + + await capture.commit(); + assert.equal(capture.failure, null); + names = (await readdir(directory)).sort((a, b) => (a < b ? -1 : 1)); + assert.deepEqual(names, [ + `0000000001-${capture.entry_id}.committed.json`, + `0000000001-${capture.entry_id}.prepared.json`, + ]); + const committed = JSON.parse(await readFile(join(directory, names[0]), "utf8")); + assert.equal(committed.schema_version, LOCAL_AUTHORITY_SHADOW_OUTBOX_COMMIT_SCHEMA); + assert.equal(committed.entry_id, capture.entry_id); + + const second = await beginLeaseOutboxEntry({ + runtime_root: root, + goal_id: GOAL, + lease_directory: leaseDirectory, + write_class: "task_lease_renew", + operation_id: null, + previous_lease: planned, + planned_lease: { ...planned, version: 3, updated_at: "t3" }, + }); + assert.equal(second.seq, 2); + const failing = await beginLeaseOutboxEntry({ + runtime_root: root, + goal_id: GOAL, + lease_directory: leaseDirectory, + write_class: "task_lease_renew", + operation_id: null, + previous_lease: null, + planned_lease: { goal_id: GOAL }, + }); + assert.equal(failing.entry_id, null); + assert.equal(failing.failure?.reason_code, "outbox_prepare_failed"); +}); + +test("entry identity derivation agrees byte-for-byte with the Python outbox module", async () => { + const python = process.env.LOOPX_TEST_PYTHON ?? "python3"; + const script = [ + "from loopx.control_plane.coordination.local_authority_shadow_outbox import entry_identity", + "from loopx.control_plane.coordination.local_authority_shadow_projection import sha256_digest", + `print(entry_identity(goal_id='${GOAL}', partition='leases', seq=7, source_ref='sha256:${HEX}'))`, + "print(sha256_digest({'handoff_mode': 'hard_lease', 'todos': [{'todo_id': 'todo-a', 'status': 'open'}], 'leases': []}))", + ].join("\n"); + let stdout: string; + try { + ({ stdout } = await execFileAsync(python, ["-c", script], { + cwd: join(import.meta.dirname, "..", ".."), + env: { ...process.env, PYTHONPATH: join(import.meta.dirname, "..", "..") }, + })); + } catch { + return; // Python without the loopx package: the Python suite pins the same fixture. + } + const [pythonEntryId, pythonHeadDigest] = stdout.trim().split("\n"); + assert.equal(pythonEntryId, outboxEntryIdentity(GOAL, "leases", 7, `sha256:${HEX}`)); + assert.equal( + pythonHeadDigest, + localAuthorityShadowHeadDigest({ + handoff_mode: "hard_lease", + todos: [{ todo_id: "todo-a", status: "open" }], + leases: [], + }), + ); +}); diff --git a/tests/control_plane_ts/task_lease_lifecycle.test.ts b/tests/control_plane_ts/task_lease_lifecycle.test.ts index e8073a7c3b..4886f5942f 100644 --- a/tests/control_plane_ts/task_lease_lifecycle.test.ts +++ b/tests/control_plane_ts/task_lease_lifecycle.test.ts @@ -598,6 +598,10 @@ test("fence close write failure leaves the lease unchanged and unlocks", async ( test("user-gate auto-acquire returns a persistent fence that can be closed", async (t) => { const root = await workspace(t); + const runtimeShadow = { + schema_version: "loopx_coordination_runtime_shadow_binding_v0", + provider: "file_v0", + }; const facts = await authority(root, { todos: [{ todo_id: "todo_target", @@ -615,10 +619,15 @@ test("user-gate auto-acquire returns a persistent fence that can be closed", asy idempotency_key: null, expected_version: null, allow_user_gate_auto_acquire: true, + runtime_shadow: runtimeShadow, }), { now: () => ACQUIRE_NOW }, ); assert.equal(checked.ok, true); + const autoCapture = checked.coordination_runtime_shadow_capture as Record; + assert.equal(typeof autoCapture.entry_id, "string"); + assert.equal(autoCapture.seq, 1); + assert.equal(autoCapture.failure, null); const fence = checked.fence as Record; assert.equal(fence.auto_acquired, true); assert.equal((await lease(root)).status, "active"); @@ -635,10 +644,15 @@ test("user-gate auto-acquire returns a persistent fence that can be closed", asy fence_owner: "agent-a", fence_idempotency_key: (await lease(root)).idempotency_key, fence_expected_version: (await lease(root)).version, + runtime_shadow: runtimeShadow, }), }, { now: () => new Date("2026-09-01T03:01:00.000Z") }); assert.equal(closed.ok, true); assert.equal(closed.released, true); + const closeCapture = closed.coordination_runtime_shadow_capture as Record; + assert.equal(typeof closeCapture.entry_id, "string"); + assert.equal(closeCapture.seq, 2); + assert.equal(closeCapture.failure, null); assert.equal((await lease(root)).status, "released"); }); diff --git a/tsconfig.control-plane.json b/tsconfig.control-plane.json index 4af88a3954..02230c3239 100644 --- a/tsconfig.control-plane.json +++ b/tsconfig.control-plane.json @@ -30,6 +30,7 @@ "loopx/control_plane/coordination/nokv_authority_store.ts", "loopx/control_plane/coordination/nokv_jsonl_transport.ts", "loopx/control_plane/coordination/local_authority_shadow.ts", + "loopx/control_plane/coordination/local_authority_shadow_outbox.ts", "loopx/control_plane/coordination/postgresql_authority_store.ts", "loopx/control_plane/agents/delivery_workspace.ts", "loopx/control_plane/goals/vision_checkpoint.ts", @@ -70,6 +71,7 @@ "tests/control_plane_ts/coordination_runtime_shadow.test.ts", "tests/control_plane_ts/coordination_projection.test.ts", "tests/control_plane_ts/local_authority_runtime.test.ts", + "tests/control_plane_ts/local_authority_shadow_outbox.test.ts", "tests/control_plane_ts/authority_store_conformance.ts", "tests/control_plane_ts/nokv_authority_store.test.ts", "tests/control_plane_ts/nokv_jsonl_transport.test.ts",