From f3da9005d4343b01d2e80cb249e750bb387b7601 Mon Sep 17 00:00:00 2001 From: wchwawa Date: Sat, 19 Sep 2026 14:25:11 +1000 Subject: [PATCH 1/3] feat(coordination): accept seed routing and report SDK capability mismatches in the NoKV helper Signed-off-by: wchwawa --- examples/nokv-authority-store/README.md | 16 +- .../coordination/nokv_jsonl_helper.py | 58 +++++- .../nokv_jsonl_transport.test.ts | 40 +++- ...nokv_stage2a_qualification_harness.test.ts | 4 + tests/fixtures/nokv_fake_sdk/nokv/__init__.py | 6 + tests/test_nokv_jsonl_helper.py | 176 ++++++++++++++++++ 6 files changed, 284 insertions(+), 16 deletions(-) diff --git a/examples/nokv-authority-store/README.md b/examples/nokv-authority-store/README.md index e8ed9d31e6..96f601e37a 100644 --- a/examples/nokv-authority-store/README.md +++ b/examples/nokv-authority-store/README.md @@ -53,9 +53,19 @@ run is Stage 2A single-node storage conformance evidence only. ## Inputs Use a current NoKV Python environment. Keep the client configuration in an -ignored local file; do not commit credentials. Static routing is valid for a -single-node NoKV deployment—etcd is not required by this probe. The following -shape is illustrative: +ignored local file; do not commit credentials. The helper accepts three routing +kinds and passes each to the matching `RoutingConfig` constructor of the +installed SDK: `etcd` and `static` (the 0.11.0 release wheel) and `seeds` +(`{"kind": "seeds", "endpoints": ["IP:PORT", ...]}`, the NoKV +metadata-runtimes line, which names serving owners directly and drops the etcd +constructor). Static routing is valid for a single-node NoKV deployment; etcd +is not required by this probe. A routing kind the installed wheel cannot build +fails the open handshake with `nokv_sdk_capability_mismatch` before any client +is constructed, so a seeds configuration against a 0.11.0 wheel (or an etcd +configuration against a metadata-runtimes wheel) is reported as the wrong +wheel, not as an outage. The `ready` handshake echoes `nokv_protocol_schema`: +the SDK's `WORKSPACE_PROTOCOL_SCHEMA` when the wheel exports one, otherwise +`null` (the 0.11.0 release does not). The following shape is illustrative: ```json { diff --git a/loopx/control_plane/coordination/nokv_jsonl_helper.py b/loopx/control_plane/coordination/nokv_jsonl_helper.py index bbcf8094d9..8e11ecd406 100644 --- a/loopx/control_plane/coordination/nokv_jsonl_helper.py +++ b/loopx/control_plane/coordination/nokv_jsonl_helper.py @@ -36,6 +36,13 @@ } ) _ETCD_ROUTING_KEYS = frozenset({"kind", "endpoints", "key_prefix", "lease_ttl_seconds"}) +# Seed routing names one or more serving NoKV owners directly (numeric IP:port); +# the SDK rejects hostnames, empty lists and unspecified ports itself. +_SEEDS_ROUTING_KEYS = frozenset({"kind", "endpoints"}) +# Every routing kind this helper can express. A kind outside this set is a +# configuration error; a kind inside it that the installed SDK cannot build is +# a capability mismatch between the wheel and the configuration. +_ROUTING_KINDS = frozenset({"etcd", "seeds", "static"}) _STATIC_ROUTING_KEYS = frozenset( { "kind", @@ -67,6 +74,14 @@ class RequestError(ValueError): """The JSON-lines caller violated the raw storage protocol.""" +class SdkCapabilityMismatch(RequestError): + """The installed NoKV SDK lacks the constructor this configuration needs. + + Raised only after the configuration itself validated, so the caller can + tell "wrong wheel for this routing kind" apart from "invalid config". + """ + + class ProviderProtocolError(RuntimeError): """The NoKV SDK returned a shape that violates its reviewed contract.""" @@ -420,6 +435,8 @@ def build_client(config_value: object) -> Any: _require_exact_keys(config, _CONFIG_KEYS, "config") routing_value = _mapping(config.get("routing"), "routing") routing_kind = _required_string(routing_value, "kind") + if routing_kind not in _ROUTING_KINDS: + raise RequestError(f"unsupported routing kind {routing_kind!r}") if routing_kind == "etcd": _require_exact_keys(routing_value, _ETCD_ROUTING_KEYS, "routing") routing_arguments: tuple[object, ...] = ( @@ -443,7 +460,8 @@ def build_client(config_value: object) -> Any: _generation(routing_value.get("owner_epoch"), "owner_epoch"), ) else: - raise RequestError(f"unsupported routing kind {routing_kind!r}") + _require_exact_keys(routing_value, _SEEDS_ROUTING_KEYS, "routing") + routing_arguments = (_string_list(routing_value, "endpoints"),) object_value = _mapping(config.get("object_store"), "object_store") object_kind = _required_string(object_value, "kind") @@ -516,12 +534,17 @@ def build_client(config_value: object) -> Any: except AttributeError as error: raise RequestError("the NoKV Python SDK surface is incomplete") from error - try: - routing = ( - RoutingConfig.etcd(*routing_arguments) - if routing_kind == "etcd" - else RoutingConfig.static(*routing_arguments) + # The kind was checked against _ROUTING_KINDS above, so this attribute + # lookup never reaches an arbitrary caller-chosen name. The 0.11.0 release + # wheel provides etcd/static; the metadata-runtimes line provides seeds. + routing_constructor = getattr(RoutingConfig, routing_kind, None) + if not callable(routing_constructor): + raise SdkCapabilityMismatch( + "the installed NoKV Python SDK does not provide " + f"RoutingConfig.{routing_kind}" ) + try: + routing = routing_constructor(*routing_arguments) except (TypeError, ValueError) as error: raise RequestError("NoKV routing configuration is invalid") from error try: @@ -550,6 +573,19 @@ def build_client(config_value: object) -> Any: raise RequestError("NoKV client configuration is invalid") from error +def _sdk_protocol_schema() -> str | None: + """Return the wire schema the imported SDK declares, if it declares one. + + The 0.11.0 release wheel has no such attribute; newer wheels export + ``WORKSPACE_PROTOCOL_SCHEMA`` so a deployment can compare it against the + server before trusting a nominally equal ``__version__``. + """ + schema = getattr(sys.modules.get("nokv"), "WORKSPACE_PROTOCOL_SCHEMA", None) + if isinstance(schema, str) and schema and schema.strip() == schema: + return schema + return None + + def main() -> int: first = sys.stdin.readline() try: @@ -561,6 +597,15 @@ def main() -> int: client = build_client(values.get("config")) except json.JSONDecodeError as error: result = _failure(None, "failed", "invalid_json", error) + except SdkCapabilityMismatch as error: + result = _failure( + _request_id(value.get("request_id")) + if isinstance(value, Mapping) + else None, + "failed", + "nokv_sdk_capability_mismatch", + error, + ) except RequestError as error: result = _failure( _request_id(value.get("request_id")) @@ -592,6 +637,7 @@ def main() -> int: "ready", nokv_sdk_version=QUALIFIED_NOKV_SDK_VERSION, nokv_api_version=QUALIFIED_NOKV_API_VERSION, + nokv_protocol_schema=_sdk_protocol_schema(), ), sort_keys=True, separators=(",", ":"), diff --git a/tests/control_plane_ts/nokv_jsonl_transport.test.ts b/tests/control_plane_ts/nokv_jsonl_transport.test.ts index 54641a31b3..02a536437d 100644 --- a/tests/control_plane_ts/nokv_jsonl_transport.test.ts +++ b/tests/control_plane_ts/nokv_jsonl_transport.test.ts @@ -21,17 +21,22 @@ const FAKE_SDK_ROOT = fileURLToPath( new URL("../fixtures/nokv_fake_sdk", import.meta.url), ); -async function openSdkHelper() { +const ETCD_ROUTING = { + kind: "etcd", + endpoints: ["http://127.0.0.1:2379"], + key_prefix: "/nokv/control", + lease_ttl_seconds: 10, +}; +// Seed routing names serving owners directly; it is the routing kind of the +// NoKV metadata-runtimes line, which drops the etcd constructor. +const SEEDS_ROUTING = { kind: "seeds", endpoints: ["127.0.0.1:7750"] }; + +async function openSdkHelper(routing: Record = ETCD_ROUTING) { return await NoKVJsonLinesTransport.open({ argv: [PYTHON, SDK_HELPER], config: { root_id: "0".repeat(32), - routing: { - kind: "etcd", - endpoints: ["http://127.0.0.1:2379"], - key_prefix: "/nokv/control", - lease_ttl_seconds: 10, - }, + routing, object_store: { kind: "memory" }, }, env: { @@ -201,3 +206,24 @@ test("NoKV AuthorityStore preserves helper protocol failure as failed, not missi assert.equal(loaded.reason_code, "provider_protocol_violation"); } }); + +test("JSON-lines transport opens the real helper with seed routing", async () => { + const transport = await openSdkHelper(SEEDS_ROUTING); + try { + const identity = await transport.storeIdentity("authority-workbench"); + assert.equal(identity.status, "available"); + if (identity.status !== "available") throw new Error("unreachable"); + assert.equal(identity.store_identity, `nokv:authority-workbench:${"a".repeat(32)}`); + } finally { + await transport.close(); + } +}); + +test("JSON-lines transport surfaces an unknown routing kind as a typed protocol failure", async () => { + await assert.rejects( + openSdkHelper({ kind: "gossip", endpoints: ["127.0.0.1:7750"] }), + (error: unknown) => + error instanceof NoKVTransportProtocolError && /routing kind/.test(error.message), + ); +}); + diff --git a/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts b/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts index 5b290895cd..258e58ee41 100644 --- a/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts +++ b/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts @@ -50,6 +50,10 @@ class RoutingConfig: def etcd(*values): return ("etcd", values) + @staticmethod + def seeds(*values): + return ("seeds", values) + @staticmethod def static(*values): return ("static", values) diff --git a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py index 4ad7970ef2..52ce1920a0 100644 --- a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py +++ b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py @@ -7,6 +7,12 @@ class RoutingConfig: + # Union of the two real wheels the helper is qualified against: the 0.11.0 + # release provides etcd/static, the metadata-runtimes line provides seeds. + @staticmethod + def seeds(endpoints: list[str]) -> object: + return ("seeds", endpoints) + @staticmethod def etcd(endpoints: list[str], key_prefix: str, lease_ttl_seconds: int) -> object: return ("etcd", endpoints, key_prefix, lease_ttl_seconds) diff --git a/tests/test_nokv_jsonl_helper.py b/tests/test_nokv_jsonl_helper.py index b67bf1d467..a17f8551de 100644 --- a/tests/test_nokv_jsonl_helper.py +++ b/tests/test_nokv_jsonl_helper.py @@ -12,6 +12,7 @@ from loopx.control_plane.coordination.nokv_jsonl_helper import ( ClientAdmissionUnavailable, RequestError, + SdkCapabilityMismatch, build_client, handle_request, main, @@ -586,5 +587,180 @@ def test_open_handshake_reports_the_qualified_sdk_contract( "request_id": "open-a", "status": "ready", "nokv_api_version": 1, + "nokv_protocol_schema": None, + "nokv_sdk_version": "0.11.0", + } + + +def _sdk_module(routing: object, **overrides: Any) -> types.SimpleNamespace: + module = types.SimpleNamespace( + __version__="0.11.0", + API_VERSION=1, + Client=lambda **_kwargs: object(), + ObjectStoreConfig=types.SimpleNamespace(memory=lambda: object()), + RoutingConfig=routing, + ) + for name, value in overrides.items(): + setattr(module, name, value) + return module + + +def _seeds_config(**routing_extra: Any) -> dict[str, Any]: + return { + "root_id": "a" * 32, + "routing": {"kind": "seeds", "endpoints": ["127.0.0.1:7750"], **routing_extra}, + "object_store": {"kind": "memory"}, + } + + +def test_seeds_route_uses_the_sdk_seeds_constructor_with_exact_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seeds_calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def seeds(*args: Any) -> object: + seeds_calls.append(args) + return object() + + monkeypatch.setitem(sys.modules, "nokv", _sdk_module(RoutingConfig)) + + build_client(_seeds_config()) + assert seeds_calls == [(["127.0.0.1:7750"],)] + + for invalid in ( + _seeds_config(key_prefix="/nokv/control"), + {**_seeds_config(), "routing": {"kind": "seeds"}}, + {**_seeds_config(), "routing": {"kind": "seeds", "endpoints": []}}, + ): + with pytest.raises(RequestError): + build_client(invalid) + assert len(seeds_calls) == 1 + + +@pytest.mark.parametrize( + ("routing_config", "sdk_routing"), + [ + ( + {"kind": "seeds", "endpoints": ["127.0.0.1:7750"]}, + types.SimpleNamespace( + etcd=lambda *_args: object(), static=lambda *_args: object() + ), + ), + ( + { + "kind": "etcd", + "endpoints": ["http://unused.invalid"], + "key_prefix": "/nokv/control", + "lease_ttl_seconds": 10, + }, + types.SimpleNamespace(seeds=lambda *_args: object()), + ), + ], +) +def test_routing_kind_the_sdk_cannot_build_is_a_typed_capability_mismatch( + monkeypatch: pytest.MonkeyPatch, + routing_config: dict[str, Any], + sdk_routing: types.SimpleNamespace, +) -> None: + constructed: list[str] = [] + module = _sdk_module( + sdk_routing, + Client=lambda **_kwargs: constructed.append("client"), + ) + module.ObjectStoreConfig = types.SimpleNamespace( + memory=lambda: constructed.append("object_store") + ) + monkeypatch.setitem(sys.modules, "nokv", module) + + with pytest.raises(SdkCapabilityMismatch) as raised: + build_client( + { + "root_id": "a" * 32, + "routing": routing_config, + "object_store": {"kind": "memory"}, + } + ) + + assert isinstance(raised.value, RequestError) + assert f"RoutingConfig.{routing_config['kind']}" in str(raised.value) + assert "127.0.0.1" not in str(raised.value) + assert "unused.invalid" not in str(raised.value) + assert constructed == [] + + +def test_unknown_routing_kind_is_invalid_config_not_a_capability_mismatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _sdk_module(types.SimpleNamespace(seeds=lambda *_args: object())) + monkeypatch.setitem(sys.modules, "nokv", module) + + with pytest.raises(RequestError) as raised: + build_client( + {**_seeds_config(), "routing": {"kind": "gossip", "endpoints": ["x"]}} + ) + assert not isinstance(raised.value, SdkCapabilityMismatch) + + +def test_open_handshake_reports_capability_mismatch_as_a_typed_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _sdk_module(types.SimpleNamespace(etcd=lambda *_args: object())) + monkeypatch.setitem(sys.modules, "nokv", module) + incoming = io.StringIO( + json.dumps( + {"request_id": "open-b", "operation": "open", "config": _seeds_config()} + ) + + "\n" + ) + outgoing = io.StringIO() + monkeypatch.setattr(sys, "stdin", incoming) + monkeypatch.setattr(sys, "stdout", outgoing) + + assert main() == 2 + response = json.loads(outgoing.getvalue().splitlines()[0]) + assert response["request_id"] == "open-b" + assert response["status"] == "failed" + assert response["reason_code"] == "nokv_sdk_capability_mismatch" + assert "RoutingConfig.seeds" in response["reason"] + assert "127.0.0.1" not in response["reason"] + + +@pytest.mark.parametrize( + ("declared", "expected"), + [ + ("nokv.workspace.rpc.v10", "nokv.workspace.rpc.v10"), + (None, None), + (10, None), + ("", None), + ], +) +def test_open_handshake_echoes_only_a_well_formed_sdk_protocol_schema( + monkeypatch: pytest.MonkeyPatch, + declared: object, + expected: str | None, +) -> None: + extra: dict[str, Any] = {} + if declared is not None: + extra["WORKSPACE_PROTOCOL_SCHEMA"] = declared + module = _sdk_module(types.SimpleNamespace(seeds=lambda *_args: object()), **extra) + monkeypatch.setitem(sys.modules, "nokv", module) + incoming = io.StringIO( + json.dumps( + {"request_id": "open-c", "operation": "open", "config": _seeds_config()} + ) + + "\n" + ) + outgoing = io.StringIO() + monkeypatch.setattr(sys, "stdin", incoming) + monkeypatch.setattr(sys, "stdout", outgoing) + + assert main() == 0 + assert json.loads(outgoing.getvalue().splitlines()[0]) == { + "request_id": "open-c", + "status": "ready", + "nokv_api_version": 1, + "nokv_protocol_schema": expected, "nokv_sdk_version": "0.11.0", } From d8cc665a39a3b3281624ebb4e35465ca396e0f7f Mon Sep 17 00:00:00 2001 From: wchwawa Date: Sat, 19 Sep 2026 14:43:56 +1000 Subject: [PATCH 2/3] feat(testing): let the NoKV live rows route through seeds and pin the SDK wire schema Signed-off-by: wchwawa --- examples/nokv-authority-store/README.md | 2 +- .../live-qualification.ts | 21 +- examples/nokv-shadow-provider/README.md | 7 +- examples/nokv-shadow-provider/live_e2e.py | 75 +++++++- examples/shared-goal-authority-e2e/README.md | 6 +- .../coordination/nokv_jsonl_transport.ts | 38 +++- .../testing/authority_e2e_ladder.py | 93 ++++++++- .../test_shared_goal_authority_e2e.py | 86 +++++++++ .../nokv_jsonl_transport.test.ts | 47 ++++- tests/fixtures/nokv_fake_sdk/nokv/__init__.py | 17 +- tests/test_nokv_live_e2e_routing.py | 179 ++++++++++++++++++ 11 files changed, 542 insertions(+), 29 deletions(-) create mode 100644 tests/test_nokv_live_e2e_routing.py diff --git a/examples/nokv-authority-store/README.md b/examples/nokv-authority-store/README.md index 96f601e37a..889f06554d 100644 --- a/examples/nokv-authority-store/README.md +++ b/examples/nokv-authority-store/README.md @@ -136,7 +136,7 @@ unfenced, pre-existing, or unreadable state exits nonzero with a compact JSON reason; provider stderr, endpoints, credentials, and raw SDK errors are not copied into that result. A successful JSON report includes `"qualification_scope":"stage_2a_single_node_store_conformance"`, -`"nokv_sdk_version":"0.11.0"`, and `"nokv_api_version":1`. The two version +`"nokv_sdk_version":"0.11.0"`, `"nokv_api_version":1`, and `"nokv_protocol_schema"` (the wheel's `WORKSPACE_PROTOCOL_SCHEMA`, or `null` for the 0.11.0 release, which exports none). The two version fields are the helper's admission constants: the helper refuses to open a client for any other SDK version or API version, so a successful report implies them, but they are not values read back from the NoKV server. The report is Stage diff --git a/examples/nokv-authority-store/live-qualification.ts b/examples/nokv-authority-store/live-qualification.ts index 8301f67aac..801043e161 100644 --- a/examples/nokv-authority-store/live-qualification.ts +++ b/examples/nokv-authority-store/live-qualification.ts @@ -25,6 +25,7 @@ import { type NoKVStoreIdentityResult, } from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; import { + NoKVHelperOpenRefusedError, NoKVJsonLinesTransport, } from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; @@ -48,6 +49,8 @@ export class QualificationFailure extends Error { export interface QualificationTransport extends NoKVBlobTransport { close(): Promise; + /** Wire schema declared by the helper's SDK; absent or `null` when unknown. */ + readonly sdkProtocolSchema?: string | null; } export interface QualificationOptions { @@ -71,6 +74,8 @@ export interface QualificationReport { availability_or_ha_proven: false; nokv_sdk_version: typeof QUALIFIED_NOKV_SDK_VERSION; nokv_api_version: typeof QUALIFIED_NOKV_API_VERSION; + /** `null` when the wheel exports no `WORKSPACE_PROTOCOL_SCHEMA` (0.11.0). */ + nokv_protocol_schema: string | null; } export interface QualificationSequenceResult { @@ -499,12 +504,16 @@ export async function exerciseQualificationSequence( export async function qualifyNoKVAuthorityStore( options: QualificationOptions, ): Promise { - const sequence = await exerciseQualificationSequence(options, async () => - await NoKVJsonLinesTransport.open({ + let protocolSchema: string | null = null; + const sequence = await exerciseQualificationSequence(options, async () => { + const transport = await NoKVJsonLinesTransport.open({ argv: qualificationHelperArgv(options.python_executable), config: options.client_config, request_timeout_ms: options.request_timeout_ms, - })); + }); + if (protocolSchema === null) protocolSchema = transport.sdkProtocolSchema; + return transport; + }); return { schema_version: REPORT_SCHEMA, qualification_scope: QUALIFICATION_SCOPE, @@ -515,6 +524,7 @@ export async function qualifyNoKVAuthorityStore( availability_or_ha_proven: false, nokv_sdk_version: QUALIFIED_NOKV_SDK_VERSION, nokv_api_version: QUALIFIED_NOKV_API_VERSION, + nokv_protocol_schema: protocolSchema, }; } @@ -610,6 +620,11 @@ async function main(): Promise { } else if (error instanceof NoKVTransportUnavailableError) { reasonCode = "nokv_backend_unavailable"; reason = "NoKV backend or SDK helper is unavailable"; + } else if (error instanceof NoKVHelperOpenRefusedError) { + // The helper's own typed code (for example nokv_sdk_capability_mismatch); + // its human-readable reason may echo configuration and stays out of stderr. + reasonCode = error.reasonCode; + reason = "NoKV SDK helper refused the open handshake"; } else if (error instanceof NoKVTransportProtocolError) { reasonCode = "nokv_transport_protocol_failed"; reason = "NoKV SDK helper violated the transport protocol"; diff --git a/examples/nokv-shadow-provider/README.md b/examples/nokv-shadow-provider/README.md index 509331ca83..3386804b31 100644 --- a/examples/nokv-shadow-provider/README.md +++ b/examples/nokv-shadow-provider/README.md @@ -145,8 +145,11 @@ shared lifecycle scenarios (including renew, reclaim after grace, stale-fence rejection, atomic completion/successor, competition, replay, lost response, retention, and revision advancement) through the production `CoordinationAuthorityExecutor` against the file-backed control provider and, -when `NOKV_COORDINATION_LIVE=1` and the stack variables are set, this NoKV -provider. One NoKV-only row performs a real commit/snapshot/restore and proves +when `NOKV_COORDINATION_LIVE=1`, the stack variables and exactly one routing +group are set (`NOKV_SEEDS` for a seed-routed owner, or `NOKV_ETCD` plus +`NOKV_ETCD_PREFIX` for the 0.11.0 etcd control path), this NoKV provider. A +routing kind the installed `nokv` wheel cannot build, or both groups set at +once, is reported as a typed unverified reason, not as a failed row. One NoKV-only row performs a real commit/snapshot/restore and proves the restored lineage fails closed as `store_lineage_mismatch`. Without a reachable stack the NoKV rows report unverified and the script stays green, so it is evidence tooling, not a merge gate. diff --git a/examples/nokv-shadow-provider/live_e2e.py b/examples/nokv-shadow-provider/live_e2e.py index a12cbf6a01..fcea151b1b 100644 --- a/examples/nokv-shadow-provider/live_e2e.py +++ b/examples/nokv-shadow-provider/live_e2e.py @@ -324,15 +324,72 @@ def file_matrix(root: Path) -> dict: ) -def nokv_matrix() -> tuple[dict | None, str | None]: +NOKV_SEEDS_VARIABLE = "NOKV_SEEDS" +NOKV_ETCD_VARIABLES = ("NOKV_ETCD", "NOKV_ETCD_PREFIX") + + +def nokv_routing(nokv, env) -> tuple[object | None, dict | None, tuple[str, str] | None]: + """Build the SDK routing from exactly one routing variable group. + + ``NOKV_SEEDS`` (comma-separated ``IP:PORT``) selects seed routing, the kind + of the NoKV metadata-runtimes line; ``NOKV_ETCD`` plus ``NOKV_ETCD_PREFIX`` + selects etcd routing, the kind of the 0.11.0 release line. Both set is a + configuration error, and a kind the installed wheel cannot build is a + capability mismatch, never an outage. Returns ``(routing, sdk_facts, + None)`` or ``(None, None, (reason, reason_code))``; the facts carry counts + and kinds only, never endpoint values. + """ + seeds_raw = env.get(NOKV_SEEDS_VARIABLE) + etcd_complete = all(env.get(name) for name in NOKV_ETCD_VARIABLES) + if seeds_raw and etcd_complete: + return None, None, ( + "NOKV_SEEDS and NOKV_ETCD/NOKV_ETCD_PREFIX are both set; choose one routing kind", + "nokv_routing_env_ambiguous", + ) + if seeds_raw: + kind = "seeds" + seeds = [item.strip() for item in seeds_raw.split(",") if item.strip()] + arguments: tuple = (seeds,) + elif etcd_complete: + kind = "etcd" + arguments = ([env["NOKV_ETCD"]], env["NOKV_ETCD_PREFIX"], 10) + else: + return None, None, ( + "neither NOKV_SEEDS nor NOKV_ETCD/NOKV_ETCD_PREFIX is set", + "nokv_routing_env_missing", + ) + constructor = getattr(nokv.RoutingConfig, kind, None) + if not callable(constructor): + return None, None, ( + f"the installed nokv SDK does not provide RoutingConfig.{kind}", + "nokv_sdk_capability_mismatch", + ) + try: + routing = constructor(*arguments) + except (TypeError, ValueError): + return None, None, ("NoKV routing configuration is invalid", "nokv_routing_invalid") + schema = getattr(nokv, "WORKSPACE_PROTOCOL_SCHEMA", None) + facts = { + "version": getattr(nokv, "__version__", None), + "api_version": getattr(nokv, "API_VERSION", None), + "protocol_schema": schema if isinstance(schema, str) and schema else None, + "routing_kind": kind, + "seed_count": len(arguments[0]) if kind == "seeds" else None, + } + return routing, facts, None + + +def nokv_matrix() -> tuple[dict | None, tuple[str, str] | None, dict | None]: if os.environ.get("NOKV_COORDINATION_LIVE") != "1": - return None, "NOKV_COORDINATION_LIVE unset" + return None, ("NOKV_COORDINATION_LIVE unset", "nokv_live_flag_unset"), None try: import nokv except ImportError: - return None, "nokv SDK not installed" + return None, ("nokv SDK not installed", "nokv_sdk_missing"), None env = os.environ - routing = nokv.RoutingConfig.etcd([env["NOKV_ETCD"]], env["NOKV_ETCD_PREFIX"], 10) + routing, sdk_facts, skip = nokv_routing(nokv, env) + if routing is None: + return None, skip, None def make_client(): objects = nokv.ObjectStoreConfig.s3( @@ -367,7 +424,7 @@ def make_provider(goal_id): rows["restored_lineage_fails_closed"] = _restored_lineage_fails_closed( make_client ) - return rows, None + return rows, None, sdk_facts def _restored_lineage_fails_closed(make_client) -> bool: @@ -435,11 +492,15 @@ def main() -> int: matrix: dict[str, dict] = {} with tempfile.TemporaryDirectory() as root: matrix["file_provider"] = file_matrix(Path(root)) - nokv_rows, skip_reason = nokv_matrix() + nokv_rows, skip, sdk_facts = nokv_matrix() if nokv_rows is None: - matrix["nokv_provider"] = {"unverified": skip_reason} + assert skip is not None + reason, reason_code = skip + matrix["nokv_provider"] = {"unverified": reason, "reason_code": reason_code} else: matrix["nokv_provider"] = nokv_rows + # Wheel facts the ladder pins into its bindings: kinds and counts only. + matrix["nokv_sdk"] = sdk_facts or {} shared = { row: value for row, value in nokv_rows.items() diff --git a/examples/shared-goal-authority-e2e/README.md b/examples/shared-goal-authority-e2e/README.md index 0857a3c61a..acfae4b9c3 100644 --- a/examples/shared-goal-authority-e2e/README.md +++ b/examples/shared-goal-authority-e2e/README.md @@ -85,7 +85,7 @@ TypeScript store read. | --- | --- | --- | | `deterministic` | none (needs `node` on `PATH` for the CLI's TypeScript runtime and the read-back probe) | `node_missing` when the probe cannot run | | `env:postgresql` | `LOOPX_TEST_POSTGRES_URL` plus `node_modules/pg` (`npm ci`) | `postgres_url_missing`, `pg_dependency_missing`, `node_missing` | -| `env:nokv_legacy` | `NOKV_COORDINATION_LIVE=1` and `NOKV_ETCD`, `NOKV_ETCD_PREFIX`, `NOKV_ROOT_ID`, `NOKV_BUCKET`, `NOKV_OBJECT_ENDPOINT`, `NOKV_OBJECT_ROOT`, `NOKV_OBJECT_KEY`, `NOKV_OBJECT_SECRET`; the `nokv` SDK importable | `nokv_live_env_missing`, `nokv_coordination_live_not_enabled`, `nokv_sdk_missing` | +| `env:nokv_legacy` | `NOKV_COORDINATION_LIVE=1`, `NOKV_ROOT_ID`, `NOKV_BUCKET`, `NOKV_OBJECT_ENDPOINT`, `NOKV_OBJECT_ROOT`, `NOKV_OBJECT_KEY`, `NOKV_OBJECT_SECRET`, plus exactly one routing group: `NOKV_SEEDS` (comma-separated `IP:PORT` owners, the NoKV metadata-runtimes line) or `NOKV_ETCD` + `NOKV_ETCD_PREFIX` (the 0.11.0 release line); the `nokv` SDK importable | `nokv_live_env_missing` (`missing_one_of` names both routing groups), `nokv_routing_env_ambiguous`, `nokv_sdk_capability_mismatch` (the installed wheel cannot build the configured routing kind), `nokv_sdk_missing`, `nokv_matrix_unverified` | | `env:nokv_authority` | `LOOPX_NOKV_AUTHORITY_LIVE=1` (the probe writes durable test data), `LOOPX_NOKV_AUTHORITY_CONFIG_JSON` (absolute path to the ignored NoKV client configuration), `LOOPX_NOKV_AUTHORITY_PYTHON` (absolute path to the Python executable that resolves NoKV SDK 0.11.0), `LOOPX_NOKV_AUTHORITY_WORKBENCH` (an existing workbench); `node` on `PATH` | `nokv_authority_env_missing`, `loopx_nokv_authority_live_not_enabled`, `nokv_authority_config_missing`, `nokv_authority_python_missing`, `node_missing` | POSIX-only rows report `unverified/posix_only` on Windows. @@ -96,8 +96,8 @@ The report schema is `loopx_shared_goal_authority_e2e_report_v0`: `rows[]` (`status in {pass, fail, unverified}`, `reason_code`, public-safe `evidence`, `duration_ms`), `pending[]`, `summary{pass, fail, unverified, pending, executed, privacy_violations}`, `bindings{loopx_commit, loopx_tree_dirty, probe_sha256[], -nokv_client_config_sha256, nokv_sdk_version, postgres_url_sha256_prefix, -pg_package_version}` (`null` when unknown), and `exit_policy`. +nokv_client_config_sha256, nokv_sdk_version, nokv_protocol_schema, +nokv_routing_group, postgres_url_sha256_prefix, pg_package_version}` (`null` when unknown), and `exit_policy`. Exit code is `0` iff `fail == 0` and `privacy_violations == 0` and (`unverified == 0` or `--allow-unverified`) and (`pending == 0` or diff --git a/loopx/control_plane/coordination/nokv_jsonl_transport.ts b/loopx/control_plane/coordination/nokv_jsonl_transport.ts index 2986b4af31..b9c0812b71 100644 --- a/loopx/control_plane/coordination/nokv_jsonl_transport.ts +++ b/loopx/control_plane/coordination/nokv_jsonl_transport.ts @@ -31,6 +31,23 @@ export type NoKVJsonLinesProcessFactory = ( options: SpawnOptionsWithoutStdio, ) => ChildProcessWithoutNullStreams; +/** + * The helper answered the open handshake with a typed `failed` response. + * + * `reasonCode` is the helper's own code (for example + * `nokv_sdk_capability_mismatch` when the installed wheel cannot build the + * configured routing kind), so callers can classify a refused open without + * parsing the human-readable reason. + */ +export class NoKVHelperOpenRefusedError extends NoKVTransportProtocolError { + readonly reasonCode: string; + + constructor(reasonCode: string, message: string) { + super(message); + this.reasonCode = reasonCode; + } +} + export interface NoKVJsonLinesTransportOptions { /** Explicit command plus arguments, for example `[python, helper.py]`. */ argv: readonly string[]; @@ -100,6 +117,12 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { private stdoutBuffer = Buffer.alloc(0); private terminalError: Error | null = null; private closing = false; + /** + * The wire schema the helper's SDK declared in its `ready` handshake, or + * `null` when the wheel exports none (the 0.11.0 release). Informational: + * the helper's own version guard stays the admission decision. + */ + sdkProtocolSchema: string | null = null; constructor( options: NoKVJsonLinesTransportOptions, @@ -167,12 +190,17 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { ); } if (response.status !== "ready") { - throw new NoKVTransportProtocolError( - response.status === "failed" && typeof response.reason === "string" - ? response.reason - : "NoKV helper did not acknowledge its open handshake", - ); + const reason = response.status === "failed" && typeof response.reason === "string" + ? response.reason + : "NoKV helper did not acknowledge its open handshake"; + if (response.status === "failed" && typeof response.reason_code === "string" + && response.reason_code.length > 0) { + throw new NoKVHelperOpenRefusedError(response.reason_code, reason); + } + throw new NoKVTransportProtocolError(reason); } + const schema = response.nokv_protocol_schema; + transport.sdkProtocolSchema = typeof schema === "string" && schema.length > 0 ? schema : null; return transport; } catch (error) { await transport.close(); diff --git a/loopx/control_plane/testing/authority_e2e_ladder.py b/loopx/control_plane/testing/authority_e2e_ladder.py index 688683c820..02edb57de1 100644 --- a/loopx/control_plane/testing/authority_e2e_ladder.py +++ b/loopx/control_plane/testing/authority_e2e_ladder.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import importlib import json import os import subprocess @@ -95,9 +96,8 @@ POSTGRES_URL_VARIABLE = "LOOPX_TEST_POSTGRES_URL" NOKV_LIVE_FLAG = "NOKV_COORDINATION_LIVE" +# Stack variables every NoKV live row needs regardless of how the client routes. NOKV_STACK_VARIABLES: tuple[str, ...] = ( - "NOKV_ETCD", - "NOKV_ETCD_PREFIX", "NOKV_ROOT_ID", "NOKV_BUCKET", "NOKV_OBJECT_ENDPOINT", @@ -106,6 +106,16 @@ "NOKV_OBJECT_SECRET", ) NOKV_SECRET_VARIABLES: tuple[str, ...] = ("NOKV_OBJECT_KEY", "NOKV_OBJECT_SECRET") +# Exactly one routing group must be complete: seed routing (comma-separated +# ``IP:PORT`` owners, the NoKV metadata-runtimes line) or etcd routing (the +# 0.11.0 release line). Both complete is ambiguous, neither is missing. +NOKV_SEEDS_VARIABLE = "NOKV_SEEDS" +NOKV_ETCD_VARIABLES: tuple[str, ...] = ("NOKV_ETCD", "NOKV_ETCD_PREFIX") +NOKV_ROUTING_GROUPS: dict[str, tuple[str, ...]] = { + "seeds": (NOKV_SEEDS_VARIABLE,), + "etcd": NOKV_ETCD_VARIABLES, +} +NOKV_ROUTING_VARIABLES: tuple[str, ...] = (NOKV_SEEDS_VARIABLE, *NOKV_ETCD_VARIABLES) # Stage 2A qualification inputs: the probe writes durable test data into an # existing workbench, so it needs an explicit opt-in flag plus the ignored # client configuration file, the Python executable that resolves the qualified @@ -142,6 +152,9 @@ NOKV_QUALIFICATION_SCOPE = "stage_2a_single_node_store_conformance" QUALIFIED_NOKV_SDK_VERSION = "0.11.0" QUALIFIED_NOKV_API_VERSION = 1 +# Typed helper code shared by the Stage 0 matrix and the Stage 2A probe when the +# installed wheel lacks the constructor for the configured routing kind. +NOKV_SDK_CAPABILITY_MISMATCH = "nokv_sdk_capability_mismatch" PROBE_SOURCES: tuple[Path, ...] = ( LIVE_E2E_SCRIPT, TS_READBACK_PROBE, @@ -288,6 +301,9 @@ def _row_nokv_live_matrix(context: RowContext) -> RowOutcome: nokv_rows = _matrix_rows(matrix, "nokv_provider") if "unverified" in nokv_rows: reason = str(nokv_rows["unverified"]) + typed = nokv_rows.get("reason_code") + if isinstance(typed, str) and typed: + return unverified(typed) code = "nokv_sdk_missing" if "SDK" in reason else "nokv_matrix_unverified" return unverified(code) expected = {*FILE_MATRIX_ROWS, NOKV_ONLY_MATRIX_ROW} @@ -297,11 +313,22 @@ def _row_nokv_live_matrix(context: RowContext) -> RowOutcome: expect(parity.get("identical_row_outcomes") is True, "file and NoKV rows must be identical") expect(parity.get("rows") == len(FILE_MATRIX_ROWS), "parity must cover the twelve shared rows") expect(matrix["_exit_code"] == 0, "live matrix script must exit 0") + sdk = matrix.get("nokv_sdk") + sdk_facts = sdk if isinstance(sdk, dict) else {} + routing_kind = sdk_facts.get("routing_kind") + routing_group = ( + NOKV_ROUTING_GROUPS[routing_kind][0] + if isinstance(routing_kind, str) and routing_kind in NOKV_ROUTING_GROUPS + else None + ) return passed( nokv_rows=len(nokv_rows), parity_rows=parity.get("rows"), restored_lineage_fails_closed=True, script_exit_code=matrix["_exit_code"], + nokv_routing_group=routing_group, + nokv_seed_count=sdk_facts.get("seed_count"), + nokv_protocol_schema=sdk_facts.get("protocol_schema"), ) @@ -447,6 +474,10 @@ def _row_nokv_live_qualification(context: RowContext) -> RowOutcome: failure = parse_json_object(completed.stderr.strip().splitlines()[-1]) except (CliOutputError, IndexError): pass + if failure.get("reason_code") == NOKV_SDK_CAPABILITY_MISMATCH: + # The installed wheel cannot build the configured routing kind: the + # row could not run, exactly like the Stage 0 matrix reports it. + return unverified(NOKV_SDK_CAPABILITY_MISMATCH) raise RowAssertionError( f"qualification probe exited {completed.returncode}: " f"{failure.get('reason_code') or 'no typed failure on stderr'}" @@ -488,6 +519,7 @@ def _row_nokv_live_qualification(context: RowContext) -> RowOutcome: final_cursor=report.get("final_cursor"), nokv_sdk_version=report.get("nokv_sdk_version"), nokv_api_version=report.get("nokv_api_version"), + nokv_protocol_schema=report.get("nokv_protocol_schema"), config_sha256_prefix=_nokv_authority_config_sha256(config_path)[:12], workbench_sha256_prefix=sha256_hex(workbench)[:12], tenant_id=tenant_id, @@ -791,12 +823,49 @@ def gate_unverified_reason(gate: str, environ: Mapping[str, str]) -> tuple[str, missing = sorted(name for name in required if not environ.get(name)) if missing: return GATE_UNVERIFIED_REASON[gate], {"missing_variables": missing} + if gate == "env:nokv_legacy": + complete = complete_nokv_routing_groups(environ) + if not complete: + return GATE_UNVERIFIED_REASON[gate], { + "missing_one_of": [list(names) for names in NOKV_ROUTING_GROUPS.values()], + } + if len(complete) > 1: + return "nokv_routing_env_ambiguous", {"routing_kinds": complete} for flag in LIVE_OPT_IN_FLAGS: if flag in required and environ.get(flag) != "1": return f"{flag.lower()}_not_enabled", {"flag": flag} return None +def complete_nokv_routing_groups(environ: Mapping[str, str]) -> list[str]: + """Routing kinds whose every variable is set, sorted by kind name.""" + + return sorted( + kind + for kind, names in NOKV_ROUTING_GROUPS.items() + if all(environ.get(name) for name in names) + ) + + +def nokv_routing_kind(environ: Mapping[str, str]) -> str | None: + """The one selected routing kind, or ``None`` when absent or ambiguous.""" + + complete = complete_nokv_routing_groups(environ) + return complete[0] if len(complete) == 1 else None + + +def nokv_routing_group(environ: Mapping[str, str]) -> str | None: + """The selected routing group named by its first variable, for reports. + + Reports name the variable (``NOKV_SEEDS`` / ``NOKV_ETCD``) rather than the + kind: the kind literal is also a string leaf of the client configuration, + which makes it a forbidden privacy token in every report. + """ + + kind = nokv_routing_kind(environ) + return NOKV_ROUTING_GROUPS[kind][0] if kind is not None else None + + def default_forbidden_tokens(roots: Iterable[Path], environ: Mapping[str, str]) -> list[str]: """Substrings whose presence in a report is a privacy leak.""" @@ -808,7 +877,12 @@ def default_forbidden_tokens(roots: Iterable[Path], environ: Mapping[str, str]) tokens.update({str(temp_root), str(temp_root.resolve())}) tokens.add(environ.get("HOME") or str(Path.home())) tokens.update({str(REPO_ROOT), str(REPO_ROOT.resolve())}) - for name in (POSTGRES_URL_VARIABLE, *NOKV_STACK_VARIABLES, *NOKV_AUTHORITY_VARIABLES): + for name in ( + POSTGRES_URL_VARIABLE, + *NOKV_STACK_VARIABLES, + *NOKV_ROUTING_VARIABLES, + *NOKV_AUTHORITY_VARIABLES, + ): value = environ.get(name) if value: tokens.add(value) @@ -962,6 +1036,17 @@ def _nokv_sdk_version() -> str | None: return None +def _nokv_protocol_schema() -> str | None: + """The wire schema the importable SDK declares; the 0.11.0 release has none.""" + + try: + module = importlib.import_module("nokv") + except ImportError: + return None + schema = getattr(module, "WORKSPACE_PROTOCOL_SCHEMA", None) + return schema if isinstance(schema, str) and schema else None + + def collect_bindings(environ: Mapping[str, str]) -> JsonObject: """Pin what the report was produced against; ``None`` means unknown.""" @@ -974,6 +1059,8 @@ def collect_bindings(environ: Mapping[str, str]) -> JsonObject: "probe_sha256": _probe_digests(), "nokv_client_config_sha256": _nokv_client_config_digest(environ), "nokv_sdk_version": _nokv_sdk_version(), + "nokv_protocol_schema": _nokv_protocol_schema(), + "nokv_routing_group": nokv_routing_group(environ), "postgres_url_sha256_prefix": sha256_hex(postgres_url)[:12] if postgres_url else None, "pg_package_version": _pg_package_version(), } diff --git a/tests/control_plane/test_shared_goal_authority_e2e.py b/tests/control_plane/test_shared_goal_authority_e2e.py index 1b9564d298..c87463fbb1 100644 --- a/tests/control_plane/test_shared_goal_authority_e2e.py +++ b/tests/control_plane/test_shared_goal_authority_e2e.py @@ -23,6 +23,7 @@ ladder.POSTGRES_URL_VARIABLE, ladder.NOKV_LIVE_FLAG, *ladder.NOKV_STACK_VARIABLES, + *ladder.NOKV_ROUTING_VARIABLES, ladder.NOKV_AUTHORITY_LIVE_FLAG, *ladder.NOKV_AUTHORITY_VARIABLES, ) @@ -403,3 +404,88 @@ def test_list_prints_rows_and_pending_declarations( assert [row["id"] for row in stage_listing["rows"]] == list(STAGE_2C2_ROW_IDS) assert [row["id"] for row in stage_listing["pending"]] == list(PENDING_ROW_IDS) assert {row["stage"] for row in stage_listing["pending"]} == {"2c2"} + + +def test_nokv_legacy_gate_requires_exactly_one_routing_group() -> None: + core = {ladder.NOKV_LIVE_FLAG: "1"} + core.update({name: f"stack-{index}" for index, name in enumerate(ladder.NOKV_STACK_VARIABLES)}) + missing_one_of = ( + "nokv_live_env_missing", + {"missing_one_of": [["NOKV_SEEDS"], ["NOKV_ETCD", "NOKV_ETCD_PREFIX"]]}, + ) + assert ladder.gate_unverified_reason("env:nokv_legacy", core) == missing_one_of + assert ladder.nokv_routing_kind(core) is None + + seeds = {**core, "NOKV_SEEDS": "127.0.0.1:7750,127.0.0.1:7751"} + assert ladder.gate_unverified_reason("env:nokv_legacy", seeds) is None + assert ladder.nokv_routing_kind(seeds) == "seeds" + + etcd = {**core, "NOKV_ETCD": "http://127.0.0.1:2379", "NOKV_ETCD_PREFIX": "/nokv/control"} + assert ladder.gate_unverified_reason("env:nokv_legacy", etcd) is None + assert ladder.nokv_routing_kind(etcd) == "etcd" + + partial = {**core, "NOKV_ETCD": "http://127.0.0.1:2379"} + assert ladder.gate_unverified_reason("env:nokv_legacy", partial) == missing_one_of + + both = {**seeds, **etcd} + assert ladder.gate_unverified_reason("env:nokv_legacy", both) == ( + "nokv_routing_env_ambiguous", + {"routing_kinds": ["etcd", "seeds"]}, + ) + assert ladder.nokv_routing_kind(both) is None + assert ladder.collect_bindings(both)["nokv_routing_group"] is None + # Reports carry the variable name: the kind literal is also a client-config + # string leaf and therefore a forbidden privacy token. + assert ladder.collect_bindings(seeds)["nokv_routing_group"] == "NOKV_SEEDS" + assert ladder.collect_bindings(etcd)["nokv_routing_group"] == "NOKV_ETCD" + + # Seed and etcd endpoint values are privacy tokens like every other stack value. + tokens = ladder.default_forbidden_tokens([], both) + assert "127.0.0.1:7750,127.0.0.1:7751" in tokens + assert "http://127.0.0.1:2379" in tokens + + # The opt-in flag is still checked after the routing group is complete. + assert ladder.gate_unverified_reason( + "env:nokv_legacy", {**seeds, ladder.NOKV_LIVE_FLAG: "0"} + ) == ("nokv_coordination_live_not_enabled", {"flag": ladder.NOKV_LIVE_FLAG}) + + +def test_stage_2a_row_reports_a_wrong_wheel_as_unverified( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The probe's typed capability mismatch is a missing stack, not a failed row.""" + + import subprocess + + config = tmp_path / "client.json" + config.write_text("{}", encoding="utf-8") + python = tmp_path / "python" + python.write_text("", encoding="utf-8") + environ = { + ladder.NOKV_AUTHORITY_LIVE_FLAG: "1", + ladder.NOKV_AUTHORITY_CONFIG_VARIABLE: str(config), + ladder.NOKV_AUTHORITY_PYTHON_VARIABLE: str(python), + ladder.NOKV_AUTHORITY_WORKBENCH_VARIABLE: "workbench-a", + } + monkeypatch.setattr(ladder, "node_executable", lambda: "/usr/bin/false") + stderr = json.dumps( + {"schema_version": "x", "ok": False, "reason_code": "nokv_sdk_capability_mismatch", "reason": "r"} + ) + monkeypatch.setattr( + ladder.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 1, stdout="", stderr=stderr + "\n"), + ) + outcome = ladder._row_nokv_live_qualification(ladder.RowContext(root=tmp_path, environ=environ)) + assert outcome.status == "unverified" + assert outcome.reason_code == "nokv_sdk_capability_mismatch" + + other = json.dumps({"schema_version": "x", "ok": False, "reason_code": "qualification_failed", "reason": "r"}) + monkeypatch.setattr( + ladder.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 1, stdout="", stderr=other + "\n"), + ) + with pytest.raises(ladder.RowAssertionError, match="qualification_failed"): + ladder._row_nokv_live_qualification(ladder.RowContext(root=tmp_path, environ=environ)) + diff --git a/tests/control_plane_ts/nokv_jsonl_transport.test.ts b/tests/control_plane_ts/nokv_jsonl_transport.test.ts index 02a536437d..3924df4a47 100644 --- a/tests/control_plane_ts/nokv_jsonl_transport.test.ts +++ b/tests/control_plane_ts/nokv_jsonl_transport.test.ts @@ -7,7 +7,10 @@ import { NoKVTransportProtocolError, NoKVTransportUnavailableError, } from "../../loopx/control_plane/coordination/nokv_authority_store.ts"; -import { NoKVJsonLinesTransport } from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; +import { + NoKVHelperOpenRefusedError, + NoKVJsonLinesTransport, +} from "../../loopx/control_plane/coordination/nokv_jsonl_transport.ts"; import { registerAuthorityStoreConformance } from "./authority_store_conformance.ts"; const PYTHON = process.env.LOOPX_TEST_PYTHON ?? "python3"; @@ -31,7 +34,10 @@ const ETCD_ROUTING = { // NoKV metadata-runtimes line, which drops the etcd constructor. const SEEDS_ROUTING = { kind: "seeds", endpoints: ["127.0.0.1:7750"] }; -async function openSdkHelper(routing: Record = ETCD_ROUTING) { +async function openSdkHelper( + routing: Record = ETCD_ROUTING, + extraEnv: Record = {}, +) { return await NoKVJsonLinesTransport.open({ argv: [PYTHON, SDK_HELPER], config: { @@ -41,6 +47,7 @@ async function openSdkHelper(routing: Record = ETCD_ROUTING) { }, env: { ...process.env, + ...extraEnv, PYTHONPATH: process.env.PYTHONPATH ? `${FAKE_SDK_ROOT}:${process.env.PYTHONPATH}` : FAKE_SDK_ROOT, @@ -227,3 +234,39 @@ test("JSON-lines transport surfaces an unknown routing kind as a typed protocol ); }); +test("JSON-lines transport records the SDK wire schema from the ready handshake", async () => { + const release = await openSdkHelper(); + try { + assert.equal(release.sdkProtocolSchema, null, "the 0.11.0-shaped wheel exports no schema"); + } finally { + await release.close(); + } + const newer = await openSdkHelper(SEEDS_ROUTING, { + LOOPX_FAKE_NOKV_PROTOCOL_SCHEMA: "nokv.workspace.rpc.v10", + }); + try { + assert.equal(newer.sdkProtocolSchema, "nokv.workspace.rpc.v10"); + } finally { + await newer.close(); + } +}); + +test("JSON-lines transport types a helper open refused for a wheel that lacks the routing kind", async () => { + await assert.rejects( + openSdkHelper(SEEDS_ROUTING, { LOOPX_FAKE_NOKV_RELEASE_SHAPE: "1" }), + (error: unknown) => + error instanceof NoKVHelperOpenRefusedError + && error instanceof NoKVTransportProtocolError + && error.reasonCode === "nokv_sdk_capability_mismatch" + && /RoutingConfig\.seeds/.test(error.message) + && !/7750/.test(error.message), + ); + // The same narrowed wheel still opens with the routing kind it does provide. + const release = await openSdkHelper(ETCD_ROUTING, { LOOPX_FAKE_NOKV_RELEASE_SHAPE: "1" }); + try { + assert.equal(release.sdkProtocolSchema, null); + } finally { + await release.close(); + } +}); + diff --git a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py index 52ce1920a0..68189a7815 100644 --- a/tests/fixtures/nokv_fake_sdk/nokv/__init__.py +++ b/tests/fixtures/nokv_fake_sdk/nokv/__init__.py @@ -1,17 +1,28 @@ from __future__ import annotations +import os from typing import Any __version__ = "0.11.0" API_VERSION = 1 +# The 0.11.0 release exports no wire schema; the metadata-runtimes wheels do. +# Tests opt into the newer shape by setting this variable for the helper process. +_schema = os.environ.get("LOOPX_FAKE_NOKV_PROTOCOL_SCHEMA") +if _schema: + WORKSPACE_PROTOCOL_SCHEMA = _schema + class RoutingConfig: # Union of the two real wheels the helper is qualified against: the 0.11.0 # release provides etcd/static, the metadata-runtimes line provides seeds. - @staticmethod - def seeds(endpoints: list[str]) -> object: - return ("seeds", endpoints) + # LOOPX_FAKE_NOKV_RELEASE_SHAPE=1 narrows the fixture to the release shape + # so a seeds configuration reaches the helper's capability-mismatch path. + if os.environ.get("LOOPX_FAKE_NOKV_RELEASE_SHAPE") != "1": + + @staticmethod + def seeds(endpoints: list[str]) -> object: + return ("seeds", endpoints) @staticmethod def etcd(endpoints: list[str], key_prefix: str, lease_ttl_seconds: int) -> object: diff --git a/tests/test_nokv_live_e2e_routing.py b/tests/test_nokv_live_e2e_routing.py new file mode 100644 index 0000000000..022ffbdfe0 --- /dev/null +++ b/tests/test_nokv_live_e2e_routing.py @@ -0,0 +1,179 @@ +"""Routing selection of the live NoKV matrix script. + +``examples/nokv-shadow-provider/live_e2e.py`` builds its SDK routing from +exactly one environment group. These tests load the script as a module and +exercise ``nokv_routing`` with stand-in SDK modules so the selection, the typed +unverified reasons and the endpoint-free SDK facts are pinned without a stack. +""" + +from __future__ import annotations + +import importlib.util +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +LIVE_E2E = ( + Path(__file__).resolve().parents[1] + / "examples" + / "nokv-shadow-provider" + / "live_e2e.py" +) + + +def _load_live_e2e() -> Any: + spec = importlib.util.spec_from_file_location("nokv_live_e2e_under_test", LIVE_E2E) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sdk(routing: object, **extra: Any) -> types.SimpleNamespace: + module = types.SimpleNamespace( + __version__="0.11.0", API_VERSION=1, RoutingConfig=routing + ) + for name, value in extra.items(): + setattr(module, name, value) + return module + + +CORE = {"NOKV_ROOT_ID": "a" * 32} + + +def test_seed_routing_uses_the_seeds_constructor_and_reports_counts_only() -> None: + live = _load_live_e2e() + calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def seeds(*args: Any) -> object: + calls.append(args) + return ("seeds", *args) + + sdk = _sdk(RoutingConfig, WORKSPACE_PROTOCOL_SCHEMA="nokv.workspace.rpc.v10") + routing, facts, skip = live.nokv_routing( + sdk, {**CORE, "NOKV_SEEDS": " 127.0.0.1:7750, 127.0.0.1:7751 "} + ) + + assert skip is None + assert calls == [(["127.0.0.1:7750", "127.0.0.1:7751"],)] + assert routing == ("seeds", ["127.0.0.1:7750", "127.0.0.1:7751"]) + assert facts == { + "version": "0.11.0", + "api_version": 1, + "protocol_schema": "nokv.workspace.rpc.v10", + "routing_kind": "seeds", + "seed_count": 2, + } + assert "127.0.0.1" not in repr(facts) + + +def test_etcd_routing_keeps_the_release_shape_and_reports_no_schema() -> None: + live = _load_live_e2e() + calls: list[tuple[Any, ...]] = [] + + class RoutingConfig: + @staticmethod + def etcd(*args: Any) -> object: + calls.append(args) + return ("etcd", *args) + + routing, facts, skip = live.nokv_routing( + _sdk(RoutingConfig), + {**CORE, "NOKV_ETCD": "http://127.0.0.1:2379", "NOKV_ETCD_PREFIX": "/nokv"}, + ) + + assert skip is None + assert calls == [(["http://127.0.0.1:2379"], "/nokv", 10)] + assert routing is not None + assert facts == { + "version": "0.11.0", + "api_version": 1, + "protocol_schema": None, + "routing_kind": "etcd", + "seed_count": None, + } + + +@pytest.mark.parametrize( + ("env", "sdk_routing", "expected_code"), + [ + ( + { + "NOKV_SEEDS": "127.0.0.1:7750", + "NOKV_ETCD": "http://x", + "NOKV_ETCD_PREFIX": "/p", + }, + types.SimpleNamespace( + seeds=lambda *_a: object(), etcd=lambda *_a: object() + ), + "nokv_routing_env_ambiguous", + ), + ( + {}, + types.SimpleNamespace(seeds=lambda *_a: object()), + "nokv_routing_env_missing", + ), + ( + {"NOKV_ETCD": "http://x"}, + types.SimpleNamespace(etcd=lambda *_a: object()), + "nokv_routing_env_missing", + ), + ( + {"NOKV_SEEDS": "127.0.0.1:7750"}, + types.SimpleNamespace( + etcd=lambda *_a: object(), static=lambda *_a: object() + ), + "nokv_sdk_capability_mismatch", + ), + ( + {"NOKV_ETCD": "http://x", "NOKV_ETCD_PREFIX": "/p"}, + types.SimpleNamespace(seeds=lambda *_a: object()), + "nokv_sdk_capability_mismatch", + ), + ( + {"NOKV_SEEDS": "not-an-address"}, + types.SimpleNamespace( + seeds=lambda *_a: (_ for _ in ()).throw(ValueError("bad")) + ), + "nokv_routing_invalid", + ), + ], +) +def test_routing_selection_failures_are_typed_and_never_construct_a_client( + env: dict[str, str], sdk_routing: types.SimpleNamespace, expected_code: str +) -> None: + live = _load_live_e2e() + routing, facts, skip = live.nokv_routing(_sdk(sdk_routing), {**CORE, **env}) + + assert routing is None and facts is None + assert skip is not None + reason, code = skip + assert code == expected_code + assert "127.0.0.1" not in reason and "http://x" not in reason + + +def test_matrix_reports_a_typed_unverified_reason_without_a_stack( + monkeypatch: pytest.MonkeyPatch, +) -> None: + live = _load_live_e2e() + monkeypatch.delenv("NOKV_COORDINATION_LIVE", raising=False) + assert live.nokv_matrix() == ( + None, + ("NOKV_COORDINATION_LIVE unset", "nokv_live_flag_unset"), + None, + ) + + monkeypatch.setenv("NOKV_COORDINATION_LIVE", "1") + monkeypatch.setitem( + sys.modules, "nokv", _sdk(types.SimpleNamespace(etcd=lambda *_a: object())) + ) + for name in ("NOKV_SEEDS", "NOKV_ETCD", "NOKV_ETCD_PREFIX"): + monkeypatch.delenv(name, raising=False) + rows, skip, facts = live.nokv_matrix() + assert rows is None and facts is None + assert skip is not None and skip[1] == "nokv_routing_env_missing" From f5bdb51f5a1e0bb7c41081eecc0c1126e94606dc Mon Sep 17 00:00:00 2001 From: wchwawa Date: Tue, 13 Oct 2026 16:22:50 +1100 Subject: [PATCH 3/3] fix(nokv): keep each helper's SDK schema in the qualification report The live qualification opens three helpers but reported only the first non-null schema any of them declared, so helpers that disagreed (for example null/v9/v10 or v9/v10/v9) produced a report naming one schema for the whole run. Record every opened helper's declaration in open order as nokv_protocol_schema_observations and name nokv_protocol_schema only when all of them agree; it is null otherwise. The derivation moves into exerciseQualificationSequence so the deterministic harness covers it. The schema stays observed metadata: disagreement does not fail the probe, and no admission, CAS or pass criterion changes. The ladder row carries the observations as evidence, and the docs no longer say that only 0.11.0 declares no schema: the 0.11.1 release wheel exports none either. Signed-off-by: wchwawa --- examples/nokv-authority-store/README.md | 9 +++-- .../live-qualification.ts | 29 +++++++++----- examples/shared-goal-authority-e2e/README.md | 2 +- .../coordination/nokv_jsonl_transport.ts | 2 +- .../testing/authority_e2e_ladder.py | 1 + ...nokv_stage2a_qualification_harness.test.ts | 38 ++++++++++++++++++- 6 files changed, 66 insertions(+), 15 deletions(-) diff --git a/examples/nokv-authority-store/README.md b/examples/nokv-authority-store/README.md index 7d8507100d..8bb8a21e8b 100644 --- a/examples/nokv-authority-store/README.md +++ b/examples/nokv-authority-store/README.md @@ -26,9 +26,12 @@ wheel labelled `0.11.1` without that publication surface is refused as ## What it proves -The report records `nokv_protocol_schema` from the selected helper SDK's ready -handshake; missing declarations remain `null`. This is observed SDK metadata, -not a new qualification or write-authority gate. An open refusal retains the +The report records each helper's ready-handshake schema declaration in +`nokv_protocol_schema_observations` (open order; `null` where the SDK declares +none, as the 0.11.x releases do). `nokv_protocol_schema` names a schema only +when all three helpers declared the same one and is `null` otherwise. This is +observed SDK metadata, not a new qualification or write-authority gate: +disagreeing helpers do not fail the probe. An open refusal retains the helper's typed reason code while omitting its free-form reason from CLI stderr. The current SDK/API pin and incarnation-fenced publication checks still apply. diff --git a/examples/nokv-authority-store/live-qualification.ts b/examples/nokv-authority-store/live-qualification.ts index bc9c4196cb..5914eed3f2 100644 --- a/examples/nokv-authority-store/live-qualification.ts +++ b/examples/nokv-authority-store/live-qualification.ts @@ -74,14 +74,21 @@ export interface QualificationReport { availability_or_ha_proven: false; nokv_sdk_version: typeof QUALIFIED_NOKV_SDK_VERSION; nokv_api_version: typeof QUALIFIED_NOKV_API_VERSION; - /** `null` when the wheel exports no `WORKSPACE_PROTOCOL_SCHEMA` (0.11.0). */ nokv_protocol_schema: string | null; + nokv_protocol_schema_observations: readonly (string | null)[]; } export interface QualificationSequenceResult { checks: readonly { id: string; status: "passed" }[]; final_generation: number; final_cursor: string; + /** + * The wire schema every opened helper declared, or `null` when they did not + * all declare the same one. The 0.11.x release wheels declare none. + */ + nokv_protocol_schema: string | null; + /** Each opened helper's declared schema in open order; `null` where none. */ + nokv_protocol_schema_observations: readonly (string | null)[]; } export interface QualificationCliArguments { @@ -281,6 +288,12 @@ function freshPhysicalIdentity(): string { return randomUUID().replaceAll("-", ""); } +/** The schema every observation names; `null` when none declared one or they disagree. */ +function uniformProtocolSchema(observations: readonly (string | null)[]): string | null { + const [first = null, ...rest] = observations; + return rest.every((schema) => schema === first) ? first : null; +} + async function rawGeneration( transport: NoKVBlobTransport, store: NoKVAuthorityStore, @@ -541,10 +554,13 @@ export async function exerciseQualificationSequence( ); passed("loser_receipt_absent"); + const protocolSchemas = opened.map((transport) => transport.sdkProtocolSchema ?? null); return { checks, final_generation: 3, final_cursor: "3", + nokv_protocol_schema: uniformProtocolSchema(protocolSchemas), + nokv_protocol_schema_observations: protocolSchemas, }; } finally { await Promise.allSettled(opened.map(async (transport) => await transport.close())); @@ -555,16 +571,12 @@ export async function exerciseQualificationSequence( export async function qualifyNoKVAuthorityStore( options: QualificationOptions, ): Promise { - let protocolSchema: string | null = null; - const sequence = await exerciseQualificationSequence(options, async () => { - const transport = await NoKVJsonLinesTransport.open({ + const sequence = await exerciseQualificationSequence(options, async () => + await NoKVJsonLinesTransport.open({ argv: qualificationHelperArgv(options.python_executable), config: options.client_config, request_timeout_ms: options.request_timeout_ms, - }); - if (protocolSchema === null) protocolSchema = transport.sdkProtocolSchema; - return transport; - }); + })); return { schema_version: REPORT_SCHEMA, qualification_scope: QUALIFICATION_SCOPE, @@ -575,7 +587,6 @@ export async function qualifyNoKVAuthorityStore( availability_or_ha_proven: false, nokv_sdk_version: QUALIFIED_NOKV_SDK_VERSION, nokv_api_version: QUALIFIED_NOKV_API_VERSION, - nokv_protocol_schema: protocolSchema, }; } diff --git a/examples/shared-goal-authority-e2e/README.md b/examples/shared-goal-authority-e2e/README.md index 7b16b0709a..fa70d29cd5 100644 --- a/examples/shared-goal-authority-e2e/README.md +++ b/examples/shared-goal-authority-e2e/README.md @@ -35,7 +35,7 @@ and the transaction-bound outbox instead of requiring a second writable history. | `s0.native_file_conformance` | 0 | store_direct | deterministic | Runs the complete `authority_store.test.ts` suite against the current TS FileAuthorityStore; every selected test must pass, with no skip or missing trailer | | `s0.native_sqlite_conformance` | 0 | store_direct | deterministic | Runs the complete `sqlite_authority_store.test.ts` suite against real SQLite files and the same shared conformance contract | | `s1.cli_document_decodes_through_ts_store` | 1 | real_cli | deterministic | Explicit bootstrap plus three CLI writes load at cursor `4`; paged `scanCommitted` returns four distinct transactions in source order and `readReceipt` finds the first source write | -| `s2a.nokv_live_qualification` | 2a | store_direct | env:nokv_authority | runs the merged `examples/nokv-authority-store/live-qualification.ts --execute-live` against an existing workbench with a fresh tenant/goal pair; requires `ok=true`, the single-node store-conformance scope, every check `passed`, NoKV SDK `0.11.1` / API `1`, the two stale-incarnation fence checks (`stale_incarnation_fence_rejected`, `stale_incarnation_fence_left_generation_unchanged`), and no promotion or availability claim; evidence carries check ids, counts, and config and workbench digest prefixes, never a configuration value or the workbench name | +| `s2a.nokv_live_qualification` | 2a | store_direct | env:nokv_authority | runs the merged `examples/nokv-authority-store/live-qualification.ts --execute-live` against an existing workbench with a fresh tenant/goal pair; requires `ok=true`, the single-node store-conformance scope, every check `passed`, NoKV SDK `0.11.1` / API `1`, the two stale-incarnation fence checks (`stale_incarnation_fence_rejected`, `stale_incarnation_fence_left_generation_unchanged`), and no promotion or availability claim; evidence carries check ids, counts, the helpers' declared SDK protocol schemas, and config and workbench digest prefixes, never a configuration value or the workbench name | | `s2b.postgresql_conformance_live` | 2b | store_direct | env:postgresql | `postgresql_authority_store.integration.test.ts` under node's TAP reporter: `# pass >= 9`, `# fail 0`, `# skipped 0` | | `s2c1.retired_observation_upgrade` | 2c1 | real_cli | deterministic | Old enable rejects without writes; retained settings create no history; explicit clear/configure/bootstrap captures the next source transaction | | `s2c2.outbox_prepared_then_committed_entries` | 2c2 | real_cli | deterministic | with the maintenance lock held, `todo add` (Python) and `task-lease acquire` (TypeScript) report `drain_deferred/drain_lock_busy`, `status` shows one `committed_pending` entry per partition with one prepared record and one committed marker on disk; one `drain` delivers both (`delivered=2`), history holds the bootstrap plus two committed receipts from both writer runtimes, and the next write delivers inline at cursor `4` | diff --git a/loopx/control_plane/coordination/nokv_jsonl_transport.ts b/loopx/control_plane/coordination/nokv_jsonl_transport.ts index efc38979b7..5a784a4d4b 100644 --- a/loopx/control_plane/coordination/nokv_jsonl_transport.ts +++ b/loopx/control_plane/coordination/nokv_jsonl_transport.ts @@ -119,7 +119,7 @@ export class NoKVJsonLinesTransport implements NoKVBlobTransport { private closing = false; /** * The wire schema the helper's SDK declared in its `ready` handshake, or - * `null` when the wheel exports none (the 0.11.0 release). Informational: + * `null` when the wheel exports none (the 0.11.x releases). Informational: * the helper's own version guard stays the admission decision. */ sdkProtocolSchema: string | null = null; diff --git a/loopx/control_plane/testing/authority_e2e_ladder.py b/loopx/control_plane/testing/authority_e2e_ladder.py index 99cf7ac4ad..826c1f04ee 100644 --- a/loopx/control_plane/testing/authority_e2e_ladder.py +++ b/loopx/control_plane/testing/authority_e2e_ladder.py @@ -415,6 +415,7 @@ def _row_nokv_live_qualification(context: RowContext) -> RowOutcome: nokv_sdk_version=report.get("nokv_sdk_version"), nokv_api_version=report.get("nokv_api_version"), nokv_protocol_schema=report.get("nokv_protocol_schema"), + nokv_protocol_schema_observations=report.get("nokv_protocol_schema_observations"), config_sha256_prefix=_nokv_authority_config_sha256(config_path)[:12], workbench_sha256_prefix=sha256_hex(workbench)[:12], tenant_id=tenant_id, diff --git a/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts b/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts index 15ea6f309a..f3c41ad5f3 100644 --- a/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts +++ b/tests/control_plane_ts/nokv_stage2a_qualification_harness.test.ts @@ -106,10 +106,12 @@ const FAKE_INCARNATION = "a".repeat(32); class FakeQualificationTransport implements QualificationTransport { readonly backend: Backend; + readonly sdkProtocolSchema?: string | null; closed = false; - constructor(backend: Backend) { + constructor(backend: Backend, sdkProtocolSchema?: string | null) { this.backend = backend; + this.sdkProtocolSchema = sdkProtocolSchema; } async storeIdentity(workbench: string): Promise { @@ -258,6 +260,40 @@ test("qualification proves create, ambiguous reconciliation, contention, and fre assert.ok(opened.every((transport) => transport.closed)); }); +test("qualification keeps every helper's SDK schema and names one only when all agree", async () => { + const v9 = "nokv.workspace.rpc.v9"; + const v10 = "nokv.workspace.rpc.v10"; + const cases: { declared: (string | null | undefined)[]; schema: string | null }[] = [ + { declared: [undefined, undefined, undefined], schema: null }, + { declared: [v9, v9, v9], schema: v9 }, + { declared: [null, v9, v10], schema: null }, + { declared: [v9, v10, v9], schema: null }, + { declared: [v9, v9, null], schema: null }, + ]; + for (const { declared, schema } of cases) { + const backend: Backend = { + blob: null, + ignoreCas: false, + publishCalls: 0, + pretendAppliedWithoutWriteOnCall: null, + }; + let opened = 0; + const report = await exerciseQualificationSequence(BASE_OPTIONS, async () => + new FakeQualificationTransport(backend, declared[opened++])); + + const label = JSON.stringify(declared); + assert.equal(opened, 3, label); + assert.deepEqual( + report.nokv_protocol_schema_observations, + declared.map((value) => value ?? null), + label, + ); + assert.equal(report.nokv_protocol_schema, schema, label); + // The schema is observed metadata: disagreeing helpers do not fail the probe. + assert.equal(report.checks.length, 15, label); + } +}); + test("qualification rejects a backend that accepts a stale incarnation fence", async () => { const backend: Backend = { blob: null,