From 9248ff6784b5289a0131507bbdd415e822eb10fb Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 07:48:31 -0700 Subject: [PATCH 1/7] Report the shards only the earlier snapshot enumerates `TOPO-213` fixes the order of an ownership delta in two groups: first the entries for shards the second snapshot enumerates, then the entries for shards only the first snapshot enumerates. The Java port computed the first group alone. `OwnershipDelta.between` iterated `after.placement().shards()`, so a shard that vanished between the two snapshots produced no entry, and the Javadoc stated that omission as though it were intended. `PLACE-075` bounds the cost of a delta over "the shards the two snapshots enumerate between them", and the Python reference has always computed that union. The two implementations therefore disagreed wherever the two snapshots enumerated different shard sets, and nothing detected it: every pair in `vectors/topology/ownership-delta.json` was `slot` at one `slotCount`, where the two shard sets are always equal and the second group is always empty. The one pair that differs, at `slotCount` 6 against 12, is the `TOPO-231` incomparability refusal and computes no delta at all. The second clause of `TOPO-213` had no executable test. A `ring` pair now carries one. Adding the token `0000000000002000` divides the extent the token `0000000000003000` bounded, so the later snapshot enumerates a shard the earlier one does not; removing it folds that extent back, so the earlier snapshot enumerates one the later one does not. The removal case fails against the unrepaired port and passes against the repaired one, and the addition case passes against both, which is exactly where the two implementations diverged. This is a repair to a port rather than a change to the design. No requirement is restated, no identifier is withdrawn, and no document changes. The suite revision changes because the suite gained two cases, so `conformance/declarations/java.json` re-declares against it and `ports/java/conformance/report.txt` is regenerated from the run the declaration names. Co-Authored-By: Claude Opus 5 --- conformance/coverage.json | 6 +- conformance/declarations/java.json | 8 +- conformance/generator/generate_extra.py | 43 +++++++- conformance/manifest.json | 44 +++++++-- .../topologies/delta-ring-added.topology.json | 34 +++++++ .../delta-ring-before.topology.json | 28 ++++++ .../delta-ring-removed.topology.json | 28 ++++++ .../vectors/topology/ownership-delta.json | 97 ++++++++++++++++++- ports/java/conformance/report.txt | 20 ++-- .../core/internal/route/OwnershipDelta.java | 21 +++- 10 files changed, 295 insertions(+), 34 deletions(-) create mode 100644 conformance/topologies/delta-ring-added.topology.json create mode 100644 conformance/topologies/delta-ring-before.topology.json create mode 100644 conformance/topologies/delta-ring-removed.topology.json diff --git a/conformance/coverage.json b/conformance/coverage.json index 78f46d8..7a63468 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -735,6 +735,7 @@ "REPL-021", "REPL-025", "RING-013", + "RING-031", "RV-013", "SEC-011", "SEC-013", @@ -769,7 +770,7 @@ "TOPO-231", "TOPO-241" ], - "requirementsAtLevel": 125, + "requirementsAtLevel": 126, "requirementsWhenDeclared": 284 }, { @@ -1247,7 +1248,8 @@ "RING-031": [ "vectors/determinism/tie-breaks.json", "vectors/placement/shard-enumeration.json", - "vectors/scale/ring-1000.json" + "vectors/scale/ring-1000.json", + "vectors/topology/ownership-delta.json" ], "RV-003": [ "vectors/determinism/tie-breaks.json", diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index c9a4ec9..9054139 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "e91417161ae0de0002cf3748a9ecd969e82c6233f0d8ab97fd3e908492b24204", + "revision": "6203a8effcf62cfe7bd9416b38656d8c6c7bbae4e1885f065ffe8b7e83a688a9", "strategySurfaces": [ "directory", "rendezvous", @@ -38,12 +38,12 @@ "command": "cd ports/java && ./gradlew test", "report": "ports/java/conformance/report.txt", "vectorFiles": 67, - "vectorCases": 596, + "vectorCases": 598, "failures": 0 }, "scale": { - "wallTimeMs": 833, - "peakResidentBytes": 683659264, + "wallTimeMs": 1006, + "peakResidentBytes": 616009728, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/generator/generate_extra.py b/conformance/generator/generate_extra.py index 9c3746e..ab77dd7 100644 --- a/conformance/generator/generate_extra.py +++ b/conformance/generator/generate_extra.py @@ -380,12 +380,36 @@ def build_identity_comparison(root): "nodes": ["d", "b", "c", "a"]}] +# `TOPO-213`: a `ring` pair whose two snapshots enumerate different shard sets. Adding the token +# `0000000000002000` divides the extent the token `0000000000003000` bounded, so the later snapshot +# enumerates a shard the earlier one does not; removing it folds that extent back, so the earlier +# snapshot enumerates one the later one does not. Every other pair in this file is `slot` at a +# fixed `slotCount`, where the two shard sets are always equal and the second clause of `TOPO-213` +# is unreachable. +DELTA_RING_BEFORE = { + "formatVersion": "1.0", "topologyId": "delta-ring", "epoch": 1, + "replication": {"factor": 2}, + "strategy": {"kind": "ring", "tokenAssignment": "explicit"}, + "nodes": [{"id": "a", "tokens": ["0000000000001000", "0000000000005000"]}, + {"id": "b", "tokens": ["0000000000003000", "0000000000007000"]}], +} + +DELTA_RING_ADDED = copy.deepcopy(DELTA_RING_BEFORE) +DELTA_RING_ADDED["epoch"] = 2 +DELTA_RING_ADDED["nodes"].append({"id": "c", "tokens": ["0000000000002000"]}) + +DELTA_RING_REMOVED = copy.deepcopy(DELTA_RING_BEFORE) +DELTA_RING_REMOVED["epoch"] = 3 + + def build_ownership_delta(root): documents = {"delta-before": DELTA_BEFORE, "delta-moved": DELTA_MOVED, "delta-reordered": DELTA_REORDERED, "delta-other-slot-count": DELTA_OTHER_COUNT, "delta-other-seed": DELTA_OTHER_SEED, "delta-wide-before": DELTA_WIDE_BEFORE, "delta-wide-after": DELTA_WIDE_AFTER, - "delta-short-before": DELTA_SHORT_BEFORE, "delta-short-after": DELTA_SHORT_AFTER} + "delta-short-before": DELTA_SHORT_BEFORE, "delta-short-after": DELTA_SHORT_AFTER, + "delta-ring-before": DELTA_RING_BEFORE, "delta-ring-added": DELTA_RING_ADDED, + "delta-ring-removed": DELTA_RING_REMOVED} for name, document in documents.items(): write_json(root / ("topologies/%s.topology.json" % name), document) snapshots = {name: Snapshot(d) for name, d in documents.items()} @@ -429,6 +453,16 @@ def build_ownership_delta(root): "`TOPO-213`: the entries follow the ascending slot index `SLOT-031` enumerates, so slots " "8 and 9 precede slot 10. Ordering the shard identifiers as octets would put 10 first " "and is the divergence a `slotCount` at or below 10 cannot show."), + ("ring-token-added", "delta-ring-before", "delta-ring-added", + ["TOPO-211", "TOPO-213", "PLACE-031", "RING-031"], + "`TOPO-213`: the added token divides the extent `0000000000003000` bounded, so the " + "second snapshot enumerates `0000000000002000` and the first does not. That shard's " + "entry carries an empty before set and reports every node it gained."), + ("ring-token-removed", "delta-ring-added", "delta-ring-removed", + ["TOPO-211", "TOPO-213", "PLACE-031", "RING-031"], + "`TOPO-213`: removing the token folds `0000000000002000` into the extent that follows " + "it, so only the first snapshot enumerates that shard. Its entry carries an empty " + "after set and follows every entry for a shard the second snapshot enumerates."), ("replica-prefix-short-of-factor", "delta-short-before", "delta-short-after", ["TOPO-211", "REPL-017", "REPL-020", "SPREAD-014"], "`TOPO-211`: `strict` over one zone level admits two replicas of the three the factor " @@ -448,10 +482,11 @@ def build_ownership_delta(root): emit(root, "vectors/topology/ownership-delta.json", "topology-ownership-delta", "ownershipDelta", "The ownership delta between two snapshots, the reorder-only case that gains and loses " - "nothing, the entry order over sixteen slots, the replica set under a shortfall, and the " - "changes that make shard identity incomparable.", + "nothing, the entry order over sixteen slots, the replica set under a shortfall, the " + "`ring` pair whose two snapshots enumerate different shard sets, and the changes that " + "make shard identity incomparable.", ["TOPO-211", "TOPO-213", "TOPO-221", "TOPO-231", "TOPO-241", "PLACE-031", "SLOT-031", - "REPL-017", "REPL-020", "SPREAD-014", "SEC-013", "ERR-010"], cases) + "RING-031", "REPL-017", "REPL-020", "SPREAD-014", "SEC-013", "ERR-010"], cases) unsupported = [{ "name": "rendezvous-enumerates-no-shard", diff --git a/conformance/manifest.json b/conformance/manifest.json index 1080f87..24b1c20 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,9 +1,9 @@ { "suite": "sharder conformance suite", "revision": { - "id": "e91417161ae0de0002cf3748a9ecd969e82c6233f0d8ab97fd3e908492b24204", + "id": "6203a8effcf62cfe7bd9416b38656d8c6c7bbae4e1885f065ffe8b7e83a688a9", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", - "fileCount": 193, + "fileCount": 196, "trees": [ "vectors", "topologies", @@ -16,8 +16,8 @@ "generator": "conformance/generator", "counts": { "vectorFiles": 67, - "vectorCases": 596, - "topologyDocuments": 102, + "vectorCases": 598, + "topologyDocuments": 105, "scenarios": 22, "scenarioSteps": 303, "properties": 28, @@ -509,10 +509,10 @@ ], "surface": "the document pipeline, the snapshot lifecycle, and what is reported", "vectorFiles": 8, - "vectorCases": 133, + "vectorCases": 135, "scenarios": 1, "properties": 22, - "requirementsNamed": 125 + "requirementsNamed": 126 }, { "level": "scale", @@ -663,6 +663,30 @@ "digest": "ffdb65dd9fb5976bf5dcb760056c229d37619bd2788c6f3d490ef37bd4e8d5eb", "valid": true }, + "topologies/delta-ring-added.topology.json": { + "topologyId": "delta-ring", + "epoch": 2, + "strategy": "ring", + "nodeCount": 3, + "digest": "2042c8625badd5ccb5019d181246a66b1b017074612f589c7a3f2c8dfe2aa142", + "valid": true + }, + "topologies/delta-ring-before.topology.json": { + "topologyId": "delta-ring", + "epoch": 1, + "strategy": "ring", + "nodeCount": 2, + "digest": "425a7afcc4eb05c3403cbde0da416622e2785d02c55a590a63be99a71c90dd46", + "valid": true + }, + "topologies/delta-ring-removed.topology.json": { + "topologyId": "delta-ring", + "epoch": 3, + "strategy": "ring", + "nodeCount": 2, + "digest": "106ba6977b64947076612f7cfa874adfb2b68c2fccedcd64bfce2eab612371ab", + "valid": true + }, "topologies/delta-short-after.topology.json": { "topologyId": "delta-short", "epoch": 2, @@ -2981,16 +3005,18 @@ "level": "core", "strategies": [], "caseStrategies": [ + "ring", "slot" ], - "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, and the changes that make shard identity incomparable.", + "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, the `ring` pair whose two snapshots enumerate different shard sets, and the changes that make shard identity incomparable.", "topology": null, - "caseCount": 6, + "caseCount": 8, "requirements": [ "ERR-010", "PLACE-031", "REPL-017", "REPL-020", + "RING-031", "SEC-013", "SLOT-031", "SPREAD-014", @@ -3000,7 +3026,7 @@ "TOPO-231", "TOPO-241" ], - "sha256": "df96523b00ab2b90758bb38693770b4e54155dd040bf7e1b366e206c2ede68aa" + "sha256": "1d5d287fe69b73ca6954c3640081356ba38c67e768d82eff3cd1ff54b4a7f618" }, { "file": "vectors/validation/documents.json", diff --git a/conformance/topologies/delta-ring-added.topology.json b/conformance/topologies/delta-ring-added.topology.json new file mode 100644 index 0000000..dc80006 --- /dev/null +++ b/conformance/topologies/delta-ring-added.topology.json @@ -0,0 +1,34 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 2, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000003000", + "0000000000007000" + ] + }, + { + "id": "c", + "tokens": [ + "0000000000002000" + ] + } + ] +} diff --git a/conformance/topologies/delta-ring-before.topology.json b/conformance/topologies/delta-ring-before.topology.json new file mode 100644 index 0000000..446ae88 --- /dev/null +++ b/conformance/topologies/delta-ring-before.topology.json @@ -0,0 +1,28 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 1, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000003000", + "0000000000007000" + ] + } + ] +} diff --git a/conformance/topologies/delta-ring-removed.topology.json b/conformance/topologies/delta-ring-removed.topology.json new file mode 100644 index 0000000..0a403ff --- /dev/null +++ b/conformance/topologies/delta-ring-removed.topology.json @@ -0,0 +1,28 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 3, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000003000", + "0000000000007000" + ] + } + ] +} diff --git a/conformance/vectors/topology/ownership-delta.json b/conformance/vectors/topology/ownership-delta.json index 8c7b3ad..163f4ea 100644 --- a/conformance/vectors/topology/ownership-delta.json +++ b/conformance/vectors/topology/ownership-delta.json @@ -2,12 +2,13 @@ "vectorSet": "topology-ownership-delta", "kind": "ownershipDelta", "level": "core", - "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, and the changes that make shard identity incomparable.", + "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, the `ring` pair whose two snapshots enumerate different shard sets, and the changes that make shard identity incomparable.", "requirements": [ "ERR-010", "PLACE-031", "REPL-017", "REPL-020", + "RING-031", "SEC-013", "SLOT-031", "SPREAD-014", @@ -333,6 +334,100 @@ ] } }, + { + "name": "ring-token-added", + "requirements": [ + "TOPO-211", + "TOPO-213", + "PLACE-031", + "RING-031" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-added.topology.json", + "note": "`TOPO-213`: the added token divides the extent `0000000000003000` bounded, so the second snapshot enumerates `0000000000002000` and the first does not. That shard's entry carries an empty before set and reports every node it gained.", + "expect": { + "shardsChanged": 2, + "delta": [ + { + "shard": "0000000000001000", + "before": [ + "a", + "b" + ], + "after": [ + "a", + "c" + ], + "gained": [ + "c" + ], + "lost": [ + "b" + ] + }, + { + "shard": "0000000000002000", + "before": [], + "after": [ + "c", + "b" + ], + "gained": [ + "b", + "c" + ], + "lost": [] + } + ] + } + }, + { + "name": "ring-token-removed", + "requirements": [ + "TOPO-211", + "TOPO-213", + "PLACE-031", + "RING-031" + ], + "before": "topologies/delta-ring-added.topology.json", + "after": "topologies/delta-ring-removed.topology.json", + "note": "`TOPO-213`: removing the token folds `0000000000002000` into the extent that follows it, so only the first snapshot enumerates that shard. Its entry carries an empty after set and follows every entry for a shard the second snapshot enumerates.", + "expect": { + "shardsChanged": 2, + "delta": [ + { + "shard": "0000000000001000", + "before": [ + "a", + "c" + ], + "after": [ + "a", + "b" + ], + "gained": [ + "b" + ], + "lost": [ + "c" + ] + }, + { + "shard": "0000000000002000", + "before": [ + "c", + "b" + ], + "after": [], + "gained": [], + "lost": [ + "b", + "c" + ] + } + ] + } + }, { "name": "replica-prefix-short-of-factor", "requirements": [ diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index 39da880..81492ec 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,17 +1,17 @@ sharder conformance report, Java port -suite revision e91417161ae0de0002cf3748a9ecd969e82c6233f0d8ab97fd3e908492b24204 +suite revision 6203a8effcf62cfe7bd9416b38656d8c6c7bbae4e1885f065ffe8b7e83a688a9 - hash 2 of 2 vector files, 34 of 34 cases, 0.00s - place 45 of 45 vector files, 291 of 291 cases, 0.04s - core 8 of 8 vector files, 133 of 133 cases, 7.35s - scale 2 of 2 vector files, 2 of 2 cases, 0.83s + hash 2 of 2 vector files, 34 of 34 cases, 0.01s + place 45 of 45 vector files, 291 of 291 cases, 0.11s + core 8 of 8 vector files, 135 of 135 cases, 8.25s + scale 2 of 2 vector files, 2 of 2 cases, 1.01s failover 3 of 3 vector files, 67 of 67 cases, 0.00s - readAffinity 1 of 1 vector files, 46 of 46 cases, 0.00s + readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s fencing 3 of 3 vector files, 8 of 8 cases, 0.00s migration 3 of 3 vector files, 15 of 15 cases, 0.00s - 596 cases, 0 failures - scale wall time 0.83s - scale wall time 833 ms - peak resident size 683659264 bytes (651 MiB) + 598 cases, 0 failures + scale wall time 1.01s + scale wall time 1006 ms + peak resident size 616009728 bytes (587 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java index b028df8..35073ac 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java @@ -51,14 +51,27 @@ public static Optional incomparable(TopologyDocument before, TopologyDoc } /** - * The shards whose replica sets differ, in the enumeration order of the later snapshot. + * The shards whose replica sets differ, in the order {@code TOPO-213} fixes: first the shards + * the later snapshot enumerates, in its own enumeration order, then the shards only the + * earlier snapshot enumerates, in that snapshot's enumeration order. * - *

A shard the later snapshot does not enumerate is absent from the delta, and a shard the - * earlier one did not enumerate has an empty before set. + *

A shard the earlier snapshot did not enumerate has an empty before set, and a shard the + * later one does not enumerate has an empty after set. Neither is empty on both sides, so a + * shard that vanished reports every node it lost. Where the two snapshots enumerate the same + * shards, which is every pair under {@code slot} at one {@code slotCount}, the second group is + * empty and the order is the later snapshot's alone. */ public static List between(PlacementEngine before, PlacementEngine after) { List changes = new ArrayList<>(); - for (String shard : after.placement().shards()) { + List later = after.placement().shards(); + Set enumeratedLater = new LinkedHashSet<>(later); + List ordered = new ArrayList<>(later); + for (String shard : before.placement().shards()) { + if (!enumeratedLater.contains(shard)) { + ordered.add(shard); + } + } + for (String shard : ordered) { List was = replicas(before, shard); List now = replicas(after, shard); if (was.equals(now)) { From dc531bfd38d0c0b3599510ac8ddfae10610f641a Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 08:11:58 -0700 Subject: [PATCH 2/7] Derive a shard lineage and plan a division over it An ownership delta joins two snapshots on the shard identifier. Where an epoch changes which shards exist, that join has no answer for the shards that appeared or vanished, and their contents still have to come from somewhere. Nothing in the design answered where. The cost was reachable and silent. Two ring snapshots at factor 2 gain a node carrying one token, which divides the extent its neighbour bounded. The new shard has no entry in the earlier snapshot, so the plan named a source equal to the destination and told two nodes to copy a shard from themselves, naming neither node that held the data. Removing the token was quieter still: the shard that absorbs the folded extent keeps its replica set, so it appeared in no delta entry and received no handoff at all. This is what a node joining or leaving a ring topology did, which is the central operation of the storage walkthrough. A lineage is the second join, over the keys a shard holds rather than over its name. It is derived from the topology document, so `formatVersion` stays at 1.0, the schema is untouched, every document valid before is valid now and digests the same, and no new strategy kind is added. `range` stays withdrawn and the `SPLIT` prefix admits no further identifier. The new requirements take the `LIN` prefix on the `migration` surface, tested at the `migration` level. No conformance surface and no level is added, and no `TOPO-*`, `FENCE-*`, or `PLACE-*` requirement moves. Lineage is deliberately kept out of the ownership delta: `TOPO-211` is tested at `core`, which a port cannot decline, and enriching it would oblige every future port to implement ring interval containment before declaring anything at all. Under `ring` an extent is the token interval of `RING-020` and containment is unsigned interval arithmetic. That code sits in `core.internal.placement` rather than beside the coordinator because `verifyUnsignedComparisons` covers that package and nothing else, and a ring extent wraps past zero, which is where a signed comparison hides. The gate caught one while this was being written. Under `slot` `TOPO-231` holds `slotCount` equal, so every extent equals its counterpart and the lineage is the identity. Under `directory` a pair whose entry sets differ is refused with a named cause until directory extents are defined, which is a correct answer in place of the silent mis-plan it replaces. Under `rendezvous` there is no extent, and `MOVE-251` already refused. The property the change rests on is that an equal shard set is an equal extent set, because the shard identifier renders the geometry under every kind. Every pair the suite carried before this is such a pair, so the regenerated tree is additions only. Records: 0086 derives the lineage from extents, 0087 settles that a strategy's lineage is derived rather than declared while the integrator's storage is declared, 0088 places it in the `migration` surface, and 0091 states that the library refuses an unaligned change and reports the rest without enforcing a preference. 0022 and 0054 are amended rather than superseded, in the form 0053 gives: 0054 was right to withdraw `range`, and one sentence of its consequences was not. Co-Authored-By: Claude Opus 5 --- conformance/coverage.json | 110 ++++++- conformance/declarations/java.json | 10 +- conformance/driver/python/run_suite.py | 32 ++ conformance/generator/generate_extra.py | 153 +++++++++- conformance/generator/sharder_ref/handoff.py | 26 +- conformance/generator/sharder_ref/lineage.py | 214 ++++++++++++++ conformance/manifest.json | 113 +++++++- .../delta-ring-unaligned.topology.json | 33 +++ .../lineage-directory-before.topology.json | 39 +++ .../lineage-directory-refined.topology.json | 57 ++++ conformance/vectors/errors/taxonomy.json | 4 +- conformance/vectors/migration/lineage.json | 258 +++++++++++++++++ .../vectors/migration/plan-construction.json | 107 +++++++ .../observability/inventory-migration.json | 17 +- docs/README.md | 4 +- docs/design/05-glossary.md | 13 + docs/design/10-specification.md | 179 +++++++++++- docs/design/30-conformance.md | 24 ++ docs/design/adr/0022-range-split-lineage.md | 9 + .../adr/0054-range-strategy-withdrawal.md | 8 + .../0086-shard-lineage-derived-from-extent.md | 104 +++++++ ...capability-derived-rather-than-declared.md | 86 ++++++ ...88-lineage-inside-the-migration-surface.md | 54 ++++ ...0091-lineage-classification-as-a-signal.md | 65 +++++ ports/java/USER-GUIDE.md | 76 ++++- ports/java/conformance/report.txt | 18 +- .../core/internal/migrate/Handoff.java | 19 ++ .../core/internal/migrate/MigrationPlan.java | 60 ++-- .../core/internal/observe/Observability.java | 4 + .../core/internal/placement/ShardExtents.java | 273 ++++++++++++++++++ .../sharder/error/ErrorCode.java | 2 +- .../sharder/conformance/ConformanceSuite.java | 2 + .../sharder/conformance/CoreVectors.java | 65 +++++ 33 files changed, 2150 insertions(+), 88 deletions(-) create mode 100644 conformance/generator/sharder_ref/lineage.py create mode 100644 conformance/topologies/delta-ring-unaligned.topology.json create mode 100644 conformance/topologies/lineage-directory-before.topology.json create mode 100644 conformance/topologies/lineage-directory-refined.topology.json create mode 100644 conformance/vectors/migration/lineage.json create mode 100644 conformance/vectors/migration/plan-construction.json create mode 100644 docs/design/adr/0086-shard-lineage-derived-from-extent.md create mode 100644 docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md create mode 100644 docs/design/adr/0088-lineage-inside-the-migration-surface.md create mode 100644 docs/design/adr/0091-lineage-classification-as-a-signal.md create mode 100644 ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java diff --git a/conformance/coverage.json b/conformance/coverage.json index 7a63468..453cea9 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -1,11 +1,30 @@ { "specification": "docs/design/10-specification.md", - "statedRequirements": 656, - "coveredRequirements": 454, - "uncoveredRequirements": 202, - "coveragePercent": 69, + "statedRequirements": 680, + "coveredRequirements": 468, + "uncoveredRequirements": 212, + "coveragePercent": 68, "identifiersNamedButNotStated": [], "byPrefix": [ + { + "prefix": "LIN", + "section": "", + "stated": 24, + "covered": 14, + "uncovered": [ + "LIN-001", + "LIN-002", + "LIN-003", + "LIN-005", + "LIN-015", + "LIN-023", + "LIN-031", + "LIN-032", + "LIN-034", + "LIN-043" + ], + "percent": 58 + }, { "prefix": "CFG", "section": "Configuration surface", @@ -965,6 +984,20 @@ "HEALTH-041", "HEALTH-043", "HEALTH-048", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-021", + "LIN-022", + "LIN-033", + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045", "MOVE-001", "MOVE-011", "MOVE-021", @@ -1036,8 +1069,8 @@ "TOPO-221", "TOPO-231" ], - "requirementsAtLevel": 80, - "requirementsWhenDeclared": 404 + "requirementsAtLevel": 94, + "requirementsWhenDeclared": 418 } ], "coveredBy": { @@ -1484,7 +1517,8 @@ ], "ERR-050": [ "scenarios/rebase-drops-a-handoff.json", - "vectors/formulas/migration-rate.json" + "vectors/formulas/migration-rate.json", + "vectors/migration/lineage.json" ], "MOVE-171": [ "properties/properties.json", @@ -1633,6 +1667,59 @@ "KEY-044": [ "vectors/keytransform/prefix-fields-2.json" ], + "LIN-004": [ + "vectors/migration/lineage.json" + ], + "LIN-006": [ + "vectors/migration/lineage.json" + ], + "LIN-007": [ + "vectors/migration/lineage.json" + ], + "LIN-011": [ + "vectors/migration/lineage.json" + ], + "LIN-012": [ + "vectors/migration/lineage.json" + ], + "LIN-013": [ + "vectors/migration/lineage.json" + ], + "LIN-014": [ + "vectors/migration/lineage.json" + ], + "LIN-021": [ + "vectors/migration/lineage.json" + ], + "LIN-022": [ + "vectors/migration/lineage.json" + ], + "LIN-033": [ + "vectors/migration/lineage.json" + ], + "MOVE-241": [ + "scenarios/handoff-happy-path.json", + "vectors/migration/lineage.json", + "vectors/topology/ownership-delta-unsupported.json" + ], + "TOPO-231": [ + "scenarios/plan-superseded-by-new-epoch.json", + "scenarios/rebase-drops-a-handoff.json", + "vectors/migration/lineage.json", + "vectors/topology/ownership-delta.json" + ], + "LIN-041": [ + "vectors/migration/plan-construction.json" + ], + "LIN-042": [ + "vectors/migration/plan-construction.json" + ], + "LIN-044": [ + "vectors/migration/plan-construction.json" + ], + "LIN-045": [ + "vectors/migration/plan-construction.json" + ], "PROP-010": [ "properties/properties.json", "vectors/movement/add-one-node.json" @@ -2231,10 +2318,6 @@ "vectors/spread/degradation-ladder.json", "vectors/spread/relaxation-stages.json" ], - "MOVE-241": [ - "scenarios/handoff-happy-path.json", - "vectors/topology/ownership-delta-unsupported.json" - ], "MOVE-251": [ "vectors/topology/ownership-delta-unsupported.json" ], @@ -2257,11 +2340,6 @@ "scenarios/handoff-happy-path.json", "vectors/topology/ownership-delta.json" ], - "TOPO-231": [ - "scenarios/plan-superseded-by-new-epoch.json", - "scenarios/rebase-drops-a-handoff.json", - "vectors/topology/ownership-delta.json" - ], "TOPO-241": [ "vectors/topology/ownership-delta.json" ], diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index 9054139..b3861a2 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "6203a8effcf62cfe7bd9416b38656d8c6c7bbae4e1885f065ffe8b7e83a688a9", + "revision": "3f56d0a573f1aaf7f364802e43a2bb0d7682eb08a8e7632eb89b86728c682070", "strategySurfaces": [ "directory", "rendezvous", @@ -37,13 +37,13 @@ "run": { "command": "cd ports/java && ./gradlew test", "report": "ports/java/conformance/report.txt", - "vectorFiles": 67, - "vectorCases": 598, + "vectorFiles": 69, + "vectorCases": 608, "failures": 0 }, "scale": { - "wallTimeMs": 1006, - "peakResidentBytes": 616009728, + "wallTimeMs": 899, + "peakResidentBytes": 591892480, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/driver/python/run_suite.py b/conformance/driver/python/run_suite.py index 97d97b0..89e5045 100644 --- a/conformance/driver/python/run_suite.py +++ b/conformance/driver/python/run_suite.py @@ -389,6 +389,36 @@ def run_ownership_delta(root, payload): compare(case["name"] + ".delta", rows, case["expect"]["delta"]) +def run_lineage(root, payload): + from sharder_ref import lineage + from sharder_ref.handoff import replica_set + for case in payload["cases"]: + before = load_topology(root, payload, case["before"]) + after = load_topology(root, payload, case["after"]) + replicas = lambda s, shard: replica_set(s, shard, lambda x: x.factor) + if case["expect"].get("lineageComputed") is False: + try: + lineage.classify(before, after, replicas) + except lineage.Refused as refusal: + compare(case["name"] + ".cause", refusal.cause, + case["expect"]["condition"]["cause"]) + continue + raise AssertionError("%s: a lineage was computed where one is refused" % case["name"]) + rows = lineage.classify(before, after, replicas) + compare(case["name"] + ".lineage", rows, case["expect"]["lineage"]) + + +def run_plan_construction(root, payload): + from sharder_ref import lineage + from sharder_ref.handoff import replica_set + for case in payload["cases"]: + before = load_topology(root, payload, case["before"]) + after = load_topology(root, payload, case["after"]) + replicas = lambda s, shard: replica_set(s, shard, lambda x: x.factor) + handoffs = lineage.plan_handoffs(before, after, replicas) + compare(case["name"] + ".handoffs", handoffs, case["expect"]["handoffs"]) + + def run_pin_shard(root, payload): for case in payload["cases"]: snapshot = load_topology(root, payload, payload["topology"]) @@ -544,6 +574,8 @@ def run_publication_events(root, payload): "errorTaxonomy": run_error_taxonomy, "defaults": run_defaults, "ownershipDelta": run_ownership_delta, + "lineage": run_lineage, + "planConstruction": run_plan_construction, "pinShard": run_pin_shard, "readAffinity": run_read_affinity, "propertyWitness": run_property_witness, diff --git a/conformance/generator/generate_extra.py b/conformance/generator/generate_extra.py index ab77dd7..3998bbe 100644 --- a/conformance/generator/generate_extra.py +++ b/conformance/generator/generate_extra.py @@ -10,6 +10,7 @@ import argparse import copy +import json import sys from pathlib import Path @@ -18,7 +19,7 @@ import topologies as T # noqa: E402 from generate import key_spec, write_json # noqa: E402 -from sharder_ref import handoff, placement, routing # noqa: E402 +from sharder_ref import handoff, lineage, placement, routing # noqa: E402 from sharder_ref.topology import Snapshot # noqa: E402 ENTRIES = [] @@ -75,7 +76,8 @@ def emit(root, path, vector_set, kind, description, requirements, cases, topolog (401, "planRefused", "no", "a plan cannot be built from the two snapshots and the policy", "correct the snapshots or the policy member named in cause", ["incomparableShards", "epochNotAdvancing", "strategyUnsupported", - "destinationOutsidePlacementSet", "policyInvalid", "topologyMismatch"]), + "destinationOutsidePlacementSet", "policyInvalid", "topologyMismatch", "unalignedLineage", + "lineageUnsupported"]), (402, "quiesced", "yes", "the shard is inside the cutover window", "retry after the window, which commitDeadlineMillis bounds", []), (403, "handoffFailed", "no", "a handoff reached failed", @@ -402,6 +404,152 @@ def build_identity_comparison(root): DELTA_RING_REMOVED["epoch"] = 3 +# `LIN-022`: removing the token `0000000000003000` and adding `0000000000002000` in the same epoch +# moves a boundary without dividing or folding an extent whole. The extent `(2000, 5000]` of the +# later snapshot overlaps `(1000, 3000]` of the earlier one and neither contains the other, so the +# lineage is unaligned and the plan is refused. +DELTA_RING_UNALIGNED = copy.deepcopy(DELTA_RING_BEFORE) +DELTA_RING_UNALIGNED["epoch"] = 4 +DELTA_RING_UNALIGNED["nodes"] = [{"id": "a", "tokens": ["0000000000001000", "0000000000005000"]}, + {"id": "b", "tokens": ["0000000000007000"]}, + {"id": "c", "tokens": ["0000000000002000"]}] + +# `LIN-013`: a `directory` pair whose entry sets differ. A directory extent is a matcher narrowed +# by the precedence of `DIR-002`, which is decidable and not yet defined, so the pair is refused. +LINEAGE_DIR_BEFORE = { + "formatVersion": "1.0", "topologyId": "lineage-directory", "epoch": 1, + "replication": {"factor": 1}, + "strategy": {"kind": "directory", + "entries": [{"match": {"kind": "prefix", "value": "ab"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "cd"}, "nodes": ["d2"]}]}, + "nodes": [{"id": "d1"}, {"id": "d2"}], +} + +LINEAGE_DIR_REFINED = copy.deepcopy(LINEAGE_DIR_BEFORE) +LINEAGE_DIR_REFINED["epoch"] = 2 +LINEAGE_DIR_REFINED["strategy"]["entries"] = [ + {"match": {"kind": "prefix", "value": "ab0"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "ab1"}, "nodes": ["d2"]}, + {"match": {"kind": "prefix", "value": "ab"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "cd"}, "nodes": ["d2"]}, +] + + +def build_lineage(root): + """`LIN-*`: the classification over pairs of documents, and the plan built over it.""" + documents = {"lineage-directory-before": LINEAGE_DIR_BEFORE, + "lineage-directory-refined": LINEAGE_DIR_REFINED, + "delta-ring-unaligned": DELTA_RING_UNALIGNED} + for name, document in documents.items(): + write_json(root / ("topologies/%s.topology.json" % name), document) + + snap = {name: Snapshot(d) for name, d in documents.items()} + snap.update({name: Snapshot(d) for name, d in + {"delta-ring-before": DELTA_RING_BEFORE, "delta-ring-added": DELTA_RING_ADDED, + "delta-ring-removed": DELTA_RING_REMOVED, "delta-before": DELTA_BEFORE, + "delta-moved": DELTA_MOVED, + "delta-other-seed": DELTA_OTHER_SEED}.items()}) + snap["rendezvous-plain"] = Snapshot( + json.loads((root / "topologies/rendezvous-plain.topology.json").read_text())) + + def replicas(snapshot, shard): + return handoff.replica_set(snapshot, shard, lambda s: s.factor) + + def path(name): + return "topologies/%s.topology.json" % name + + cases = [] + for label, before, after, requirements, note in [ + ("ring-extent-divided", "delta-ring-before", "delta-ring-added", + ["LIN-004", "LIN-011", "LIN-021", "LIN-033"], + "`LIN-011`: the added token divides `(1000, 3000]` into `(1000, 2000]` and " + "`(2000, 3000]`, so both shards of the later snapshot are `divided` from one parent and " + "the shard whose extent did not move is `moved` because its replica set did."), + ("ring-extent-folded", "delta-ring-added", "delta-ring-removed", + ["LIN-004", "LIN-011", "LIN-021", "LIN-033"], + "`LIN-021`: removing the token folds `(1000, 2000]` into the extent that follows it, so " + "the later shard is `merged` from two parents and the shard only the earlier snapshot " + "enumerates is `folded` and follows every later entry under `LIN-033`."), + ("slot-identity", "delta-before", "delta-moved", + ["LIN-007", "LIN-012", "LIN-021"], + "`LIN-012`: `TOPO-231` holds `slotCount` equal, so the two snapshots enumerate the same " + "shards, the lineage is the identity of `LIN-007`, and a shard is `moved` or `unchanged` " + "and never divided, merged, fresh, or vacated."), + ]: + cases.append({ + "name": label, "requirements": requirements, + "before": path(before), "after": path(after), "note": note, + "expect": {"lineage": lineage.classify(snap[before], snap[after], replicas)}, + }) + + for label, before, after, requirements, cause, note in [ + ("ring-unaligned-boundary", "delta-ring-before", "delta-ring-unaligned", + ["LIN-021", "LIN-022", "ERR-050"], "unalignedLineage", + "`LIN-022`: a boundary that moves without dividing or folding an extent whole has no " + "correspondence to name, so the plan is refused and the authority publishes the change " + "as a division epoch followed by a fold epoch."), + ("directory-differing-shard-sets", "lineage-directory-before", "lineage-directory-refined", + ["LIN-013", "ERR-050"], "unalignedLineage", + "`LIN-013`: a `directory` extent is a matcher narrowed by the precedence of `DIR-002`. " + "It is decidable and not yet defined, so a pair whose shard sets differ is refused."), + ("incomparable-shard-identity", "delta-before", "delta-other-seed", + ["LIN-006", "TOPO-231", "ERR-050"], "incomparableShards", + "`LIN-006`: a lineage joins two snapshots on their extents and an ownership delta joins " + "them on their identifiers, and a change that renames every shard leaves neither one an " + "answer, so both refuse on the condition `TOPO-231` states."), + ("rendezvous-has-no-extent", "rendezvous-plain", "rendezvous-plain", + ["LIN-014", "MOVE-241", "ERR-050"], "strategyUnsupported", + "`LIN-014`: `PLACE-032` makes `shards` empty under `rendezvous`, so the kind has no " + "extent and no lineage, and `MOVE-251` refuses the plan before one is reached."), + ]: + cases.append({ + "name": label, "requirements": requirements, + "before": path(before), "after": path(after), "note": note, + "expect": {"lineageComputed": False, + "condition": {"code": 401, "name": "planRefused", "cause": cause}}, + }) + + emit(root, "vectors/migration/lineage.json", "migration-lineage", "lineage", + "The lineage classification over pairs of documents: a ring extent divided, a ring extent " + "folded, the identity lineage under `slot`, the unaligned boundary that is refused, the " + "`directory` pair whose shard sets differ, the incomparable pair, and the kind that " + "enumerates no shard.", + ["LIN-004", "LIN-006", "LIN-007", "LIN-011", "LIN-012", "LIN-013", "LIN-014", "LIN-021", + "LIN-022", "LIN-033", "TOPO-231", "MOVE-241", "ERR-050"], cases, level="migration") + + plans = [] + for label, before, after, requirements, note in [ + ("divided-source-from-the-parent", "delta-ring-before", "delta-ring-added", + ["LIN-041", "LIN-042", "LIN-045"], + "`LIN-041`: the contents of the divided shard `0000000000002000` are held by the " + "replicas of its parent `0000000000003000`, so the handoff names one of them as its " + "source. A plan built over the ownership delta alone has no entry for the parent, whose " + "replica set did not change, and names a source equal to the destination."), + ("folded-destination-outside-the-delta", "delta-ring-added", "delta-ring-removed", + ["LIN-041", "LIN-044", "LIN-045"], + "`LIN-044`: the shard that absorbs the folded extent keeps its replica set, so the " + "ownership delta reports no entry for it, and a plan built over the delta alone moves " + "nothing to the replica that does not hold the folded parent."), + ("slot-plan-over-the-identity-lineage", "delta-before", "delta-moved", + ["LIN-041", "LIN-045"], + "`LIN-007`: under the identity lineage every shard is its own parent, so the plan is the " + "one the ownership delta already produced and this case is the regression guard for it."), + ]: + plans.append({ + "name": label, "requirements": requirements, + "before": path(before), "after": path(after), "note": note, + "expect": {"handoffs": lineage.plan_handoffs(snap[before], snap[after], replicas)}, + }) + + emit(root, "vectors/migration/plan-construction.json", "migration-plan-construction", + "planConstruction", + "How a plan derives a handoff from a lineage: the source of a divided shard, the " + "destination of a fold that the ownership delta does not report, and the plan under the " + "identity lineage.", + ["LIN-041", "LIN-042", "LIN-044", "LIN-045"], plans, level="migration") + + + def build_ownership_delta(root): documents = {"delta-before": DELTA_BEFORE, "delta-moved": DELTA_MOVED, "delta-reordered": DELTA_REORDERED, "delta-other-slot-count": DELTA_OTHER_COUNT, @@ -614,6 +762,7 @@ def main(): build_defaults(root) build_identity_comparison(root) build_ownership_delta(root) + build_lineage(root) build_ring_and_pin_cases(root) print("wrote %d further vector files" % len(ENTRIES)) diff --git a/conformance/generator/sharder_ref/handoff.py b/conformance/generator/sharder_ref/handoff.py index 4d10b89..bc0a2f9 100644 --- a/conformance/generator/sharder_ref/handoff.py +++ b/conformance/generator/sharder_ref/handoff.py @@ -479,6 +479,20 @@ def rebase(self, topology_id, epoch, replica_sets, at=None, comparable=True): "rebaseInterval": self.rebase_interval()} +def replica_set(snapshot, shard, factor_of): + """`TOPO-211`: the entries of a shard's preference list whose role is `replica`. + + The replica count is the achieved count `r` of `REPL-020` rather than the configured factor, so + a shortfall shortens the prefix and the entries beyond it are the fallback tail of `REPL-013`. + Both the ownership delta and the lineage read a replica set through this one definition. + """ + from . import placement, routing as route_module + + ordering = placement.candidates_for_shard(snapshot, shard, snapshot.placement_set) + entries, r, _, _ = route_module.build_preference_list(snapshot, ordering, factor_of(snapshot)) + return [e["node"] for e in entries[:r]] + + def ownership_delta(before, after, factor_of): """`TOPO-211`: the shards whose ordered replica set differs, with nodes gained and lost. @@ -498,16 +512,8 @@ def ownership_delta(before, after, factor_of): ordered = order_after + [s for s in order_before if s not in shards_after] rows = [] for shard in ordered: - def replicas(snapshot, present): - if not present: - return [] - ordering = placement.candidates_for_shard(snapshot, shard, snapshot.placement_set) - entries, r, _, _ = route_module.build_preference_list( - snapshot, ordering, factor_of(snapshot)) - return [e["node"] for e in entries[:r]] - - was = replicas(before, shard in shards_before) - now = replicas(after, shard in shards_after) + was = replica_set(before, shard, factor_of) if shard in shards_before else [] + now = replica_set(after, shard, factor_of) if shard in shards_after else [] if was != now: rows.append({ "shard": shard, diff --git a/conformance/generator/sharder_ref/lineage.py b/conformance/generator/sharder_ref/lineage.py new file mode 100644 index 0000000..70571e9 --- /dev/null +++ b/conformance/generator/sharder_ref/lineage.py @@ -0,0 +1,214 @@ +"""Shard lineage: the correspondence between the extents of two snapshots. + +`LIN-001` through `LIN-045`. A lineage relates extents rather than shard identifiers, so it +answers where a shard's contents come from when an epoch changes which shards exist. An ownership +delta joins two snapshots on the identifier and has no answer there. + +The extent representation is per strategy, under `LIN-011` through `LIN-015`. Under `ring` an +extent is a set of half-open segments of the 64-bit hash space, which wraps at zero, so containment +and union are interval arithmetic. Under `slot` `TOPO-231` holds `slotCount` equal, so every extent +equals its counterpart and the lineage is the identity of `LIN-007`. Under `directory` this module +refuses a pair whose shard sets differ, which is the refusal `LIN-013` states. +""" + +from . import placement + +SPACE = 1 << 64 + + +class Refused(Exception): + """`ERR-050`: a plan refused with the cause the lineage names.""" + + def __init__(self, cause, detail=None): + super().__init__(cause if detail is None else "%s: %s" % (cause, detail)) + self.cause = cause + self.detail = detail + + +# ------------------------------------------------------------------ interval extents under `ring` + +def _segments(low_exclusive, high_inclusive): + """The keys `h` with `low_exclusive < h <= high_inclusive`, as half-open `[start, end)` pairs. + + `RING-020` gives the owning entry that interval, and it wraps at zero, so a wrapping interval is + two segments rather than one. + """ + start, end = (low_exclusive + 1) % SPACE, (high_inclusive + 1) % SPACE + if start == end: + return ((0, SPACE),) + if start < end: + return ((start, end),) + return ((0, end), (start, SPACE)) + + +def ring_extents(snapshot): + """`LIN-011`: each distinct token's interval, in the ascending ring order of `RING-031`.""" + tokens = [int(s, 16) for s in placement.ring_shards(snapshot)] + out = {} + for position, token in enumerate(tokens): + predecessor = tokens[position - 1] if len(tokens) > 1 else token + out[placement.hashing.hex_u64(token)] = _segments(predecessor, token) + return out + + +def _measure(segments): + return sum(end - start for start, end in segments) + + +def _intersection(first, second): + out = [] + for a_start, a_end in first: + for b_start, b_end in second: + start, end = max(a_start, b_start), min(a_end, b_end) + if start < end: + out.append((start, end)) + return tuple(sorted(out)) + + +def meets(first, second): + return bool(_intersection(first, second)) + + +def contains(outer, inner): + return _measure(_intersection(outer, inner)) == _measure(inner) + + +def equal(first, second): + return _measure(first) == _measure(second) == _measure(_intersection(first, second)) + + +# ------------------------------------------------------------------------------- the lineage + +def extents(snapshot): + """The extent of each shard the snapshot enumerates, under `LIN-011` through `LIN-014`.""" + kind = snapshot.strategy["kind"] + if kind == "ring": + return ring_extents(snapshot) + if kind == "rendezvous": + raise Refused("strategyUnsupported", "rendezvous enumerates no shard") + # `LIN-012` and `LIN-013`: an identity extent, keyed by the shard identifier itself. Under + # `slot` `TOPO-231` holds `slotCount` equal; under `directory` a differing shard set is refused + # by `classify` before an extent is compared. + return {shard: shard for shard in placement.shards(snapshot)} + + +def _identity_lineage(before, after, replicas_of): + rows = [] + for shard in placement.shards(after): + was, now = replicas_of(before, shard), replicas_of(after, shard) + rows.append({"shard": shard, "class": "unchanged" if was == now else "moved", + "parents": [shard]}) + return rows + + +def comparable(before, after): + """`TOPO-231`: the member the two snapshots differ in, or `None` where they are comparable. + + Shard identity is the join both an ownership delta and a lineage rest on, so the two joins + share one comparability rule and `LIN-006` refuses exactly where `TOPO-231` does. + """ + if before.strategy["kind"] != after.strategy["kind"]: + return "strategy.kind" + if before.key_transform != after.key_transform: + return "keyTransform" + if before.seed != after.seed: + return "hash.seed" + if before.hash_config["algorithm"] != after.hash_config["algorithm"]: + return "hash.algorithm" + if before.strategy["kind"] == "slot" \ + and before.strategy["slotCount"] != after.strategy["slotCount"]: + return "slotCount" + return None + + +def classify(before, after, replicas_of): + """`LIN-021`: each shard of the later snapshot, then each shard only the earlier one holds. + + `LIN-033` fixes that order, which is the order `TOPO-213` fixes for an ownership delta. + """ + differing = comparable(before, after) + if differing is not None: + raise Refused("incomparableShards", differing) + kind = before.strategy["kind"] + if kind == "rendezvous": + raise Refused("strategyUnsupported", "rendezvous enumerates no shard") + + shards_before, shards_after = placement.shards(before), placement.shards(after) + if kind in ("slot", "directory"): + if shards_before != shards_after: + # `LIN-013`: a directory extent is decidable and not yet defined, so the pair is + # refused rather than guessed. Under `slot` `TOPO-231` has already refused a differing + # `slotCount`, so an unequal shard set cannot arise. + raise Refused("unalignedLineage", "the two snapshots enumerate different shards") + return _identity_lineage(before, after, replicas_of) + + # `LIN-007`: an equal shard set is an equal extent set, so the lineage is the identity. + if shards_before == shards_after: + return _identity_lineage(before, after, replicas_of) + + earlier, later = extents(before), extents(after) + rows = [] + for shard in shards_after: + mine = later[shard] + parents = [s for s in shards_before if meets(earlier[s], mine)] + for parent in parents: + if not contains(earlier[parent], mine) and not contains(mine, earlier[parent]): + raise Refused("unalignedLineage", shard) + if not parents: + rows.append({"shard": shard, "class": "fresh", "parents": []}) + elif len(parents) > 1: + rows.append({"shard": shard, "class": "merged", "parents": parents}) + elif equal(earlier[parents[0]], mine): + was, now = replicas_of(before, shard), replicas_of(after, shard) + rows.append({"shard": shard, "class": "unchanged" if was == now else "moved", + "parents": parents}) + else: + rows.append({"shard": shard, "class": "divided", "parents": parents}) + + for shard in shards_before: + if shard in later: + continue + mine = earlier[shard] + children = [s for s in shards_after if meets(later[s], mine)] + if not children: + rows.append({"shard": shard, "class": "vacated", "parents": []}) + elif len(children) > 1: + rows.append({"shard": shard, "class": "split", "parents": children}) + else: + rows.append({"shard": shard, "class": "folded", "parents": children}) + return rows + + +def parents_of(before, after): + """For each shard of the later snapshot, the shards of the earlier one its extent draws from.""" + kind = before.strategy["kind"] + shards_before, shards_after = placement.shards(before), placement.shards(after) + if kind in ("slot", "directory") or shards_before == shards_after: + return {shard: [shard] for shard in shards_after} + earlier, later = extents(before), extents(after) + return {shard: [s for s in shards_before if meets(earlier[s], later[shard])] + for shard in shards_after} + + +def plan_handoffs(before, after, replicas_of): + """`LIN-041` through `LIN-045`: one handoff per parent a destination does not already hold.""" + parents = parents_of(before, after) + out = [] + for shard in placement.shards(after): + destinations = replicas_of(after, shard) + for parent in parents[shard]: + held = replicas_of(before, parent) + if not held: + continue + needing = [d for d in destinations if d not in held] + departing = [n for n in held if n not in destinations] + for position, destination in enumerate(needing): + # `LIN-045`: a replica giving the shard up is drained before one keeping it. + source = departing[position] if position < len(departing) else held[0] + out.append({"shard": shard, "sourceShard": parent, + "source": source, "destination": destination}) + for position, handoff in enumerate(out): + same = [h for h in out[:position] if h["shard"] == handoff["shard"]] + handoff["id"] = "h-%s" % handoff["shard"] if not same \ + else "h-%s-%d" % (handoff["shard"], len(same)) + return out diff --git a/conformance/manifest.json b/conformance/manifest.json index 24b1c20..a89347e 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,9 +1,9 @@ { "suite": "sharder conformance suite", "revision": { - "id": "6203a8effcf62cfe7bd9416b38656d8c6c7bbae4e1885f065ffe8b7e83a688a9", + "id": "3f56d0a573f1aaf7f364802e43a2bb0d7682eb08a8e7632eb89b86728c682070", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", - "fileCount": 196, + "fileCount": 201, "trees": [ "vectors", "topologies", @@ -15,13 +15,13 @@ "design": "docs/design/30-conformance.md", "generator": "conformance/generator", "counts": { - "vectorFiles": 67, - "vectorCases": 598, - "topologyDocuments": 105, + "vectorFiles": 69, + "vectorCases": 608, + "topologyDocuments": 108, "scenarios": 22, "scenarioSteps": 303, "properties": 28, - "requirementsNamed": 454 + "requirementsNamed": 468 }, "requirementsNamed": [ "CFG-014", @@ -189,6 +189,20 @@ "KEY-042", "KEY-043", "KEY-044", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-021", + "LIN-022", + "LIN-033", + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045", "MOVE-001", "MOVE-011", "MOVE-021", @@ -568,11 +582,11 @@ "failover" ], "surface": "the handoff coordinator and rate control", - "vectorFiles": 3, - "vectorCases": 15, + "vectorFiles": 5, + "vectorCases": 25, "scenarios": 12, "properties": 3, - "requirementsNamed": 80 + "requirementsNamed": 94 } ], "strategySurfaces": [ @@ -687,6 +701,14 @@ "digest": "106ba6977b64947076612f7cfa874adfb2b68c2fccedcd64bfce2eab612371ab", "valid": true }, + "topologies/delta-ring-unaligned.topology.json": { + "topologyId": "delta-ring", + "epoch": 4, + "strategy": "ring", + "nodeCount": 3, + "digest": "9856ceda5c5b9ca5bdd4240396b79dffc1fdb51d8c0d542017fe0a70088df6f2", + "valid": true + }, "topologies/delta-short-after.topology.json": { "topologyId": "delta-short", "epoch": 2, @@ -1031,6 +1053,22 @@ "digest": "7797a43770f84df98db19f9d11329d55243c779d050fd21cce026f985e0394bb", "valid": true }, + "topologies/lineage-directory-before.topology.json": { + "topologyId": "lineage-directory", + "epoch": 1, + "strategy": "directory", + "nodeCount": 2, + "digest": "202c08ccc95f89693fff2411d987ea83bf6e54536b0ff0434d7b77027cf2588d", + "valid": true + }, + "topologies/lineage-directory-refined.topology.json": { + "topologyId": "lineage-directory", + "epoch": 2, + "strategy": "directory", + "nodeCount": 2, + "digest": "88b5a9b7766b11d4cee599ccff278e02f6321850547b5012273d59c1a8f2899b", + "valid": true + }, "topologies/migration-epoch-1.topology.json": { "topologyId": "migration", "epoch": 1, @@ -1706,7 +1744,7 @@ "ERR-021", "REPL-025" ], - "sha256": "0c944ea2e167f309384f731a76f7d3ff8407094bb6d7fdab7f1401276ae2cd8a" + "sha256": "16468a36a940a2a1fe2028e57b2da665f2800a403f2cee3552832d8822ae2373" }, { "file": "vectors/formulas/attempt-limit.json", @@ -2000,6 +2038,59 @@ ], "sha256": "ae9cde46271a4ee1650e11cab0e94da757ea92c83b5eb5e447a50fff6e7679fe" }, + { + "file": "vectors/migration/lineage.json", + "vectorSet": "migration-lineage", + "kind": "lineage", + "level": "migration", + "strategies": [], + "caseStrategies": [ + "directory", + "rendezvous", + "ring", + "slot" + ], + "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` pair whose shard sets differ, the incomparable pair, and the kind that enumerates no shard.", + "topology": null, + "caseCount": 7, + "requirements": [ + "ERR-050", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-021", + "LIN-022", + "LIN-033", + "MOVE-241", + "TOPO-231" + ], + "sha256": "39281cb0f3216695062075b0935328864baf6aede3a19b3df5033c36e10a1182" + }, + { + "file": "vectors/migration/plan-construction.json", + "vectorSet": "migration-plan-construction", + "kind": "planConstruction", + "level": "migration", + "strategies": [], + "caseStrategies": [ + "ring", + "slot" + ], + "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, and the plan under the identity lineage.", + "topology": null, + "caseCount": 3, + "requirements": [ + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045" + ], + "sha256": "c19f77e44d8c9a6c686e0c6b508c666c3718fb2266f7755c63cee21d55aae2de" + }, { "file": "vectors/movement/add-one-node.json", "vectorSet": "movement-add-one-node", @@ -2078,7 +2169,7 @@ "OBS-020", "OBS-021" ], - "sha256": "548ce6104a96251eea8bdd586c79069f5d0bdbb9968b16426a5f170187cce189" + "sha256": "cea1227356a1ea21fd617f8a149d503ffd226757e8f7eeb34b4a904d7e1bea70" }, { "file": "vectors/observability/inventory-routing.json", diff --git a/conformance/topologies/delta-ring-unaligned.topology.json b/conformance/topologies/delta-ring-unaligned.topology.json new file mode 100644 index 0000000..2b87961 --- /dev/null +++ b/conformance/topologies/delta-ring-unaligned.topology.json @@ -0,0 +1,33 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 4, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000007000" + ] + }, + { + "id": "c", + "tokens": [ + "0000000000002000" + ] + } + ] +} diff --git a/conformance/topologies/lineage-directory-before.topology.json b/conformance/topologies/lineage-directory-before.topology.json new file mode 100644 index 0000000..9bfeb76 --- /dev/null +++ b/conformance/topologies/lineage-directory-before.topology.json @@ -0,0 +1,39 @@ +{ + "formatVersion": "1.0", + "topologyId": "lineage-directory", + "epoch": 1, + "replication": { + "factor": 1 + }, + "strategy": { + "kind": "directory", + "entries": [ + { + "match": { + "kind": "prefix", + "value": "ab" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "cd" + }, + "nodes": [ + "d2" + ] + } + ] + }, + "nodes": [ + { + "id": "d1" + }, + { + "id": "d2" + } + ] +} diff --git a/conformance/topologies/lineage-directory-refined.topology.json b/conformance/topologies/lineage-directory-refined.topology.json new file mode 100644 index 0000000..39f8c44 --- /dev/null +++ b/conformance/topologies/lineage-directory-refined.topology.json @@ -0,0 +1,57 @@ +{ + "formatVersion": "1.0", + "topologyId": "lineage-directory", + "epoch": 2, + "replication": { + "factor": 1 + }, + "strategy": { + "kind": "directory", + "entries": [ + { + "match": { + "kind": "prefix", + "value": "ab0" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "ab1" + }, + "nodes": [ + "d2" + ] + }, + { + "match": { + "kind": "prefix", + "value": "ab" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "cd" + }, + "nodes": [ + "d2" + ] + } + ] + }, + "nodes": [ + { + "id": "d1" + }, + { + "id": "d2" + } + ] +} diff --git a/conformance/vectors/errors/taxonomy.json b/conformance/vectors/errors/taxonomy.json index ca842e0..24184aa 100644 --- a/conformance/vectors/errors/taxonomy.json +++ b/conformance/vectors/errors/taxonomy.json @@ -169,7 +169,9 @@ "strategyUnsupported", "destinationOutsidePlacementSet", "policyInvalid", - "topologyMismatch" + "topologyMismatch", + "unalignedLineage", + "lineageUnsupported" ] }, { diff --git a/conformance/vectors/migration/lineage.json b/conformance/vectors/migration/lineage.json new file mode 100644 index 0000000..2e15a74 --- /dev/null +++ b/conformance/vectors/migration/lineage.json @@ -0,0 +1,258 @@ +{ + "vectorSet": "migration-lineage", + "kind": "lineage", + "level": "migration", + "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` pair whose shard sets differ, the incomparable pair, and the kind that enumerates no shard.", + "requirements": [ + "ERR-050", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-021", + "LIN-022", + "LIN-033", + "MOVE-241", + "TOPO-231" + ], + "cases": [ + { + "name": "ring-extent-divided", + "requirements": [ + "LIN-004", + "LIN-011", + "LIN-021", + "LIN-033" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-added.topology.json", + "note": "`LIN-011`: the added token divides `(1000, 3000]` into `(1000, 2000]` and `(2000, 3000]`, so both shards of the later snapshot are `divided` from one parent and the shard whose extent did not move is `moved` because its replica set did.", + "expect": { + "lineage": [ + { + "shard": "0000000000001000", + "class": "moved", + "parents": [ + "0000000000001000" + ] + }, + { + "shard": "0000000000002000", + "class": "divided", + "parents": [ + "0000000000003000" + ] + }, + { + "shard": "0000000000003000", + "class": "divided", + "parents": [ + "0000000000003000" + ] + }, + { + "shard": "0000000000005000", + "class": "unchanged", + "parents": [ + "0000000000005000" + ] + }, + { + "shard": "0000000000007000", + "class": "unchanged", + "parents": [ + "0000000000007000" + ] + } + ] + } + }, + { + "name": "ring-extent-folded", + "requirements": [ + "LIN-004", + "LIN-011", + "LIN-021", + "LIN-033" + ], + "before": "topologies/delta-ring-added.topology.json", + "after": "topologies/delta-ring-removed.topology.json", + "note": "`LIN-021`: removing the token folds `(1000, 2000]` into the extent that follows it, so the later shard is `merged` from two parents and the shard only the earlier snapshot enumerates is `folded` and follows every later entry under `LIN-033`.", + "expect": { + "lineage": [ + { + "shard": "0000000000001000", + "class": "moved", + "parents": [ + "0000000000001000" + ] + }, + { + "shard": "0000000000003000", + "class": "merged", + "parents": [ + "0000000000002000", + "0000000000003000" + ] + }, + { + "shard": "0000000000005000", + "class": "unchanged", + "parents": [ + "0000000000005000" + ] + }, + { + "shard": "0000000000007000", + "class": "unchanged", + "parents": [ + "0000000000007000" + ] + }, + { + "shard": "0000000000002000", + "class": "folded", + "parents": [ + "0000000000003000" + ] + } + ] + } + }, + { + "name": "slot-identity", + "requirements": [ + "LIN-007", + "LIN-012", + "LIN-021" + ], + "before": "topologies/delta-before.topology.json", + "after": "topologies/delta-moved.topology.json", + "note": "`LIN-012`: `TOPO-231` holds `slotCount` equal, so the two snapshots enumerate the same shards, the lineage is the identity of `LIN-007`, and a shard is `moved` or `unchanged` and never divided, merged, fresh, or vacated.", + "expect": { + "lineage": [ + { + "shard": "0", + "class": "unchanged", + "parents": [ + "0" + ] + }, + { + "shard": "1", + "class": "unchanged", + "parents": [ + "1" + ] + }, + { + "shard": "2", + "class": "unchanged", + "parents": [ + "2" + ] + }, + { + "shard": "3", + "class": "moved", + "parents": [ + "3" + ] + }, + { + "shard": "4", + "class": "moved", + "parents": [ + "4" + ] + }, + { + "shard": "5", + "class": "moved", + "parents": [ + "5" + ] + } + ] + } + }, + { + "name": "ring-unaligned-boundary", + "requirements": [ + "LIN-021", + "LIN-022", + "ERR-050" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-unaligned.topology.json", + "note": "`LIN-022`: a boundary that moves without dividing or folding an extent whole has no correspondence to name, so the plan is refused and the authority publishes the change as a division epoch followed by a fold epoch.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "unalignedLineage" + } + } + }, + { + "name": "directory-differing-shard-sets", + "requirements": [ + "LIN-013", + "ERR-050" + ], + "before": "topologies/lineage-directory-before.topology.json", + "after": "topologies/lineage-directory-refined.topology.json", + "note": "`LIN-013`: a `directory` extent is a matcher narrowed by the precedence of `DIR-002`. It is decidable and not yet defined, so a pair whose shard sets differ is refused.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "unalignedLineage" + } + } + }, + { + "name": "incomparable-shard-identity", + "requirements": [ + "LIN-006", + "TOPO-231", + "ERR-050" + ], + "before": "topologies/delta-before.topology.json", + "after": "topologies/delta-other-seed.topology.json", + "note": "`LIN-006`: a lineage joins two snapshots on their extents and an ownership delta joins them on their identifiers, and a change that renames every shard leaves neither one an answer, so both refuse on the condition `TOPO-231` states.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "incomparableShards" + } + } + }, + { + "name": "rendezvous-has-no-extent", + "requirements": [ + "LIN-014", + "MOVE-241", + "ERR-050" + ], + "before": "topologies/rendezvous-plain.topology.json", + "after": "topologies/rendezvous-plain.topology.json", + "note": "`LIN-014`: `PLACE-032` makes `shards` empty under `rendezvous`, so the kind has no extent and no lineage, and `MOVE-251` refuses the plan before one is reached.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "strategyUnsupported" + } + } + } + ] +} diff --git a/conformance/vectors/migration/plan-construction.json b/conformance/vectors/migration/plan-construction.json new file mode 100644 index 0000000..6bf81d5 --- /dev/null +++ b/conformance/vectors/migration/plan-construction.json @@ -0,0 +1,107 @@ +{ + "vectorSet": "migration-plan-construction", + "kind": "planConstruction", + "level": "migration", + "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, and the plan under the identity lineage.", + "requirements": [ + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045" + ], + "cases": [ + { + "name": "divided-source-from-the-parent", + "requirements": [ + "LIN-041", + "LIN-042", + "LIN-045" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-added.topology.json", + "note": "`LIN-041`: the contents of the divided shard `0000000000002000` are held by the replicas of its parent `0000000000003000`, so the handoff names one of them as its source. A plan built over the ownership delta alone has no entry for the parent, whose replica set did not change, and names a source equal to the destination.", + "expect": { + "handoffs": [ + { + "shard": "0000000000001000", + "sourceShard": "0000000000001000", + "source": "b", + "destination": "c", + "id": "h-0000000000001000" + }, + { + "shard": "0000000000002000", + "sourceShard": "0000000000003000", + "source": "a", + "destination": "c", + "id": "h-0000000000002000" + } + ] + } + }, + { + "name": "folded-destination-outside-the-delta", + "requirements": [ + "LIN-041", + "LIN-044", + "LIN-045" + ], + "before": "topologies/delta-ring-added.topology.json", + "after": "topologies/delta-ring-removed.topology.json", + "note": "`LIN-044`: the shard that absorbs the folded extent keeps its replica set, so the ownership delta reports no entry for it, and a plan built over the delta alone moves nothing to the replica that does not hold the folded parent.", + "expect": { + "handoffs": [ + { + "shard": "0000000000001000", + "sourceShard": "0000000000001000", + "source": "c", + "destination": "b", + "id": "h-0000000000001000" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000002000", + "source": "c", + "destination": "a", + "id": "h-0000000000003000" + } + ] + } + }, + { + "name": "slot-plan-over-the-identity-lineage", + "requirements": [ + "LIN-041", + "LIN-045" + ], + "before": "topologies/delta-before.topology.json", + "after": "topologies/delta-moved.topology.json", + "note": "`LIN-007`: under the identity lineage every shard is its own parent, so the plan is the one the ownership delta already produced and this case is the regression guard for it.", + "expect": { + "handoffs": [ + { + "shard": "3", + "sourceShard": "3", + "source": "b", + "destination": "d", + "id": "h-3" + }, + { + "shard": "4", + "sourceShard": "4", + "source": "b", + "destination": "d", + "id": "h-4" + }, + { + "shard": "5", + "sourceShard": "5", + "source": "b", + "destination": "d", + "id": "h-5" + } + ] + } + } + ] +} diff --git a/conformance/vectors/observability/inventory-migration.json b/conformance/vectors/observability/inventory-migration.json index fe4efcc..28145cc 100644 --- a/conformance/vectors/observability/inventory-migration.json +++ b/conformance/vectors/observability/inventory-migration.json @@ -81,10 +81,11 @@ "note": "`OBS-020` names each event, the severity it carries, and the payload members it carries beyond the common members of `OBS-021`. A port may carry further members and may not rename one of these. A `deduplication` member appears on the four events `OBS-024` names and on no other.", "expect": { "surface": "migration", - "eventCount": 9, + "eventCount": 10, "names": [ "sharder.migration.cutover_committed", "sharder.migration.failed", + "sharder.migration.lineage", "sharder.migration.planned", "sharder.migration.quiesce_expired", "sharder.migration.rebase_pending", @@ -94,6 +95,20 @@ "sharder.migration.superseded" ], "events": [ + { + "name": "sharder.migration.lineage", + "severity": "info", + "payload": [ + "unchanged", + "moved", + "divided", + "merged", + "fresh", + "split", + "folded", + "vacated" + ] + }, { "name": "sharder.migration.planned", "severity": "info", diff --git a/docs/README.md b/docs/README.md index 7b30082..8568647 100644 --- a/docs/README.md +++ b/docs/README.md @@ -197,10 +197,10 @@ other port carries too. | placement and weights | [0002](design/adr/0002-placement-strategy-set.md), [0003](design/adr/0003-integer-node-weights.md), [0010](design/adr/0010-shard-identifier-naming.md), [0011](design/adr/0011-derived-assignment-virtual-nodes.md), [0013](design/adr/0013-range-bounds-over-routing-key.md), [0039](design/adr/0039-placement-cost-model-and-warning-thresholds.md), [0043](design/adr/0043-assignment-mode-defaults.md), [0054](design/adr/0054-range-strategy-withdrawal.md), [0055](design/adr/0055-slot-derived-assignment-withdrawal.md), [0071](design/adr/0071-candidate-ordering-over-a-placement-ring.md), [0072](design/adr/0072-preparation-cost-reported-before-preparation.md) | | topology model and format | [0004](design/adr/0004-topology-provider-contract.md), [0005](design/adr/0005-epoch-and-version-semantics.md), [0006](design/adr/0006-failure-domain-model.md), [0008](design/adr/0008-json-canonical-serialisation.md), [0009](design/adr/0009-override-composition.md), [0038](design/adr/0038-provider-contract-in-the-specification.md), [0048](design/adr/0048-conditional-fetch-in-the-provider-contract.md) | | replication and failover | [0007](design/adr/0007-administrative-state-and-health-state.md), [0012](design/adr/0012-balance-bound-tolerances.md), [0014](design/adr/0014-read-affinity-as-a-separate-call.md), [0015](design/adr/0015-spread-degradation-algorithm.md), [0016](design/adr/0016-node-health-state-machine.md), [0017](design/adr/0017-failover-depth-and-substitution.md), [0036](design/adr/0036-spread-relaxation-ladder-direction.md), [0042](design/adr/0042-domain-path-scope.md), [0045](design/adr/0045-attempt-limit-resolution-order.md), [0046](design/adr/0046-bounded-routing-decision-surface.md), [0063](design/adr/0063-decision-api-surface-boundaries.md), [0069](design/adr/0069-per-level-occupancy-cap.md), [0070](design/adr/0070-spread-stage-feasibility-from-a-domain-count.md), [0075](design/adr/0075-affinity-path-length-refusal.md) | -| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md) | +| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md), [0086](design/adr/0086-shard-lineage-derived-from-extent.md), [0087](design/adr/0087-lineage-capability-derived-rather-than-declared.md), [0091](design/adr/0091-lineage-classification-as-a-signal.md) | | cross-cutting contracts | [0024](design/adr/0024-closed-numbered-error-taxonomy.md), [0025](design/adr/0025-observability-contract-and-explain-record.md), [0026](design/adr/0026-configuration-defaults-and-locality.md), [0027](design/adr/0027-hash-seed-exposure-and-tenancy.md), [0060](design/adr/0060-node-label-cardinality-ceiling.md), [0076](design/adr/0076-ordered-rows-in-a-precedence-table.md) | | Java binding | [0028](design/adr/0028-java-module-and-artifact-layout.md), [0029](design/adr/0029-exception-idiom-for-the-taxonomy.md), [0031](design/adr/0031-jdk-baseline.md), [0032](design/adr/0032-dependency-free-json-and-canonicalisation.md), [0033](design/adr/0033-opaque-identifier-value-types.md), [0034](design/adr/0034-lazy-candidate-traversal-surface.md), [0041](design/adr/0041-exact-product-comparison-surface.md), [0081](design/adr/0081-single-java-module.md), [0085](design/adr/0085-hook-declarations-and-refused-aborts.md) | -| conformance | [0035](design/adr/0035-manifest-driven-conformance-harness.md), [0037](design/adr/0037-specification-defect-repairs.md), [0052](design/adr/0052-conformance-level-partition.md), [0058](design/adr/0058-conformance-surfaces.md), [0059](design/adr/0059-place-conformance-level.md), [0061](design/adr/0061-suite-revision-identifier.md), [0064](design/adr/0064-hash-verification-before-generation.md), [0065](design/adr/0065-level-coverage-inside-surface-boundaries.md), [0077](design/adr/0077-scale-conformance-level.md), [0078](design/adr/0078-observability-contract-as-data.md), [0084](design/adr/0084-missing-member-as-a-validation-error.md) | +| conformance | [0035](design/adr/0035-manifest-driven-conformance-harness.md), [0037](design/adr/0037-specification-defect-repairs.md), [0052](design/adr/0052-conformance-level-partition.md), [0058](design/adr/0058-conformance-surfaces.md), [0059](design/adr/0059-place-conformance-level.md), [0061](design/adr/0061-suite-revision-identifier.md), [0064](design/adr/0064-hash-verification-before-generation.md), [0065](design/adr/0065-level-coverage-inside-surface-boundaries.md), [0077](design/adr/0077-scale-conformance-level.md), [0078](design/adr/0078-observability-contract-as-data.md), [0084](design/adr/0084-missing-member-as-a-validation-error.md), [0088](design/adr/0088-lineage-inside-the-migration-surface.md) | | requirement identifiers | [0053](design/adr/0053-requirement-withdrawal-convention.md) | | ports and declarations | [0079](design/adr/0079-repository-layout-for-multiple-ports.md), [0080](design/adr/0080-conformance-declaration-format.md), [0082](design/adr/0082-continuous-integration-and-dependency-updates.md) | | release staging | [0083](design/adr/0083-publication-as-the-last-stage.md) | diff --git a/docs/design/05-glossary.md b/docs/design/05-glossary.md index 27b4cea..95a6eb5 100644 --- a/docs/design/05-glossary.md +++ b/docs/design/05-glossary.md @@ -25,6 +25,19 @@ the design states, and the Java port renders each one. lowercase hexadecimal, so the strategy enumerates no shard extents. - **shard identifier**. The stable name of a shard within a topology, unique across the topology document. +- **extent**. The set of routing keys a snapshot maps to one shard. An extent is determined by the + topology document alone, never by sampling keys, and the extents of one snapshot are pairwise + disjoint. Under `ring` an extent is a token range, under `slot` it is the keys whose remainder is + one slot index, and under `directory` it is a matcher narrowed by the precedence of the entries + that outrank it. +- **lineage**. The correspondence between the extents of two snapshots of one topology, which + answers where a shard's contents come from when an epoch changes which shards exist. A lineage is + derived from the two documents rather than recorded in either, and it is the identity wherever + the two snapshots enumerate the same shards. +- **split**. A change in which one shard's extent becomes the extents of two or more shards of the + next epoch. The authority publishes the change; the library classifies it and plans it. +- **merge**. A change in which the extents of two or more shards become the extent of one shard of + the next epoch. A merge is the reverse of a split. - **partition**. A synonym for shard in external literature. The corpus uses shard. - **slot**. A shard produced by dividing the keyspace into a fixed count of numbered parts by modular arithmetic over the key hash. diff --git a/docs/design/10-specification.md b/docs/design/10-specification.md index 55f9dce..8b130e0 100644 --- a/docs/design/10-specification.md +++ b/docs/design/10-specification.md @@ -218,6 +218,7 @@ the rule the Withdrawn identifiers section gives. | `READ` | read routing and read affinity | Replication and failover | `readAffinity` | | `TOPO` | load pipeline, snapshots, ownership delta | Topology change and rebalancing | `routing` | | `FENCE` | fencing tokens and recipient verdicts | Topology change and rebalancing | `fencing` | +| `LIN` | shard extents, the lineage relation, and its classification | Topology change and rebalancing | `migration` | | `MOVE` | handoff states, hooks, coordination | Topology change and rebalancing | `migration` | | `RATE` | migration concurrency and backpressure | Topology change and rebalancing | `migration` | | `ERR` | the closed set of failure conditions | Error taxonomy | stated above | @@ -2912,6 +2913,172 @@ redirect is never a first attempt, so `FAIL-032` does not exempt one. Where the redirect, an implementation MUST count the refusal in `sharder.attempts.retries_refused` and MUST NOT count the redirect in `sharder.fencing.redirects`. +### Shard lineage + +An ownership delta joins two snapshots on the shard identifier. Where a shard carries the same +identifier under both, that join answers which nodes gained and lost it. Where an epoch changes +which shards exist, the join has no answer for the shards whose identifiers appeared or vanished, +and the contents of those shards still have to come from somewhere. + +A lineage is the second join, over the keys a shard holds rather than over its name. It is derived +from the topology document, it is consulted only where the two snapshots enumerate different shard +sets, and it belongs to the `migration` surface: an implementation that exposes no handoff +coordinator computes none. + +#### Shard extent + +`LIN-001`. The **extent** of a shard under a snapshot MUST be the set of routing keys that +`shardOf` maps to that shard under that snapshot. An extent MUST be determined from the topology +document alone. An implementation MUST NOT compute an extent by sampling routing keys, evaluating +`shardOf` over a generated set, or reading any value outside the document. + +`LIN-002`. Two extents are equal when they admit the same routing keys, are disjoint when they +admit no routing key in common, and the first contains the second when every routing key the second +admits is one the first admits. An implementation MUST decide each of the three from the +document rather than by enumerating keys, and `LIN-011` through `LIN-014` state how under each +strategy kind. + +`LIN-003`. The extents of the shards one snapshot enumerates MUST be pairwise disjoint. A routing +key that `shardOf` maps to no shard, which under `directory` is the no-match of `DIR-010`, belongs +to no extent, and an implementation MUST NOT treat it as belonging to one. + +#### Lineage of two snapshots + +`LIN-004`. The **lineage** of an earlier snapshot and a later snapshot MUST be the correspondence +their extents induce: for each shard of the later snapshot, the shards of the earlier one whose +extents meet its extent, and for each shard of the earlier snapshot, the shards of the later one +whose extents meet its extent. A lineage relates extents and MUST NOT be read from, or recorded in, +any member of a topology document. + +`LIN-005`. Computing a lineage MUST be a pure function of the two snapshots. It MUST NOT read +health, a clock, a handoff state, or any ownership delta. + +`LIN-006`. An implementation MUST refuse to compute a lineage between two snapshots whose shard +identity is not comparable under `TOPO-231`, and MUST report the refusal as `TOPO-231` states. +Comparability is the same condition for both joins, because a change that renames every shard +leaves neither one an answer. + +`LIN-007`. Where the two snapshots enumerate the same set of shard identifiers, the lineage MUST be +the identity: each shard's parent and child are that shard alone, and no shard is classified +`divided`, `merged`, `fresh`, `split`, `folded`, or `vacated` under `LIN-021`. Under every strategy kind the shard +identifier of `PLACE-031` renders the geometry that fixes the extent, so an equal shard set is an +equal extent set: under `ring` the identifier is the owning token, under `slot` it is the slot index +at a `slotCount` that `TOPO-231` holds equal, and under `directory` it is the matcher itself, whose +effective extent is fixed by the entry set that the shard set enumerates. + +#### Extents under each strategy + +`LIN-011`. Under `ring`, the extent of the shard whose identifier decodes to the token `t` MUST be +the half-open interval of routing key hash values from the greatest token below `t` to `t`, +ascending as unsigned 64-bit values and wrapping at zero, which is the interval `RING-020` gives the +owning ring entry. Containment and equality MUST be decided by comparing interval bounds as unsigned +64-bit values. + +`LIN-012`. Under `slot`, `TOPO-231` refuses a pair whose `slotCount` differs, so the two snapshots +enumerate the same slots and every extent is equal to its counterpart. The lineage under `slot` MUST +therefore be the identity of `LIN-007`. + +`LIN-013`. Under `directory`, an implementation MUST refuse to compute a lineage where the two +snapshots enumerate different shard sets, and MUST report the refusal under `ERR-050` with the cause +`unalignedLineage`. A directory extent is a matcher narrowed by the precedence of `DIR-002`, so it +is decidable, and this requirement states the refusal that stands until it is defined. + +`LIN-014`. Under `rendezvous`, `PLACE-032` makes `shards` empty and `MOVE-241` excludes the kind +from orchestrated migration, so there is no extent and no lineage. An implementation MUST refuse a +plan under `MOVE-251` before it reaches a lineage. + +`LIN-015`. A registered strategy outside the core set MUST supply its own lineage. An implementation +MUST NOT derive a registered strategy's extents by sampling routing keys and MUST NOT infer a +lineage from the strategy's name. Where the two snapshots enumerate different shard sets and the +strategy supplies no lineage, an implementation MUST refuse the plan under `ERR-050` with the cause +`lineageUnsupported`. + +#### Lineage classification + +`LIN-021`. An implementation MUST classify each shard of each snapshot against the other as exactly +one of these. + +A shard the later snapshot enumerates is classified against the earlier one, and a shard only the +earlier snapshot enumerates is classified against the later one, so every shard of the lineage +carries exactly one class. + +| Class | Snapshot | Condition | +|---|---|---| +| `unchanged` | later | one parent, whose extent is equal, and the replica set is equal | +| `moved` | later | one parent, whose extent is equal, and the replica set differs | +| `divided` | later | one parent, whose extent strictly contains this one | +| `merged` | later | two or more parents, whose extents this one contains | +| `fresh` | later | no parent, this extent meeting no extent of the earlier snapshot | +| `split` | earlier only | two or more children, whose extents this one contains | +| `folded` | earlier only | one child, whose extent strictly contains this one | +| `vacated` | earlier only | no child, this extent meeting no extent of the later snapshot | +| `unaligned` | either | an extent of the other snapshot that this one neither contains nor is contained by | + +`LIN-022`. An implementation MUST refuse a plan over a lineage that classifies any shard +`unaligned`, and MUST report the refusal under `ERR-050` with the cause `unalignedLineage` naming +the shard. A boundary that moves without either dividing or folding an extent whole has no +correspondence an implementation can name, and an authority publishes such a change as two epochs +instead, the first dividing every extent the change crosses and the second folding the pieces into +their destinations. Each of the two is plannable on its own. + +`LIN-023`. A classification MUST be a pure function of the two snapshots under `LIN-005`, and an +implementation MUST NOT let the order in which it enumerates shards change any shard's class. + +#### The lineage operation + +`LIN-031`. An implementation that exposes the `migration` surface MUST expose the lineage between +two snapshots of the same `topologyId` as an operation the integrator calls, answering each shard's +class under `LIN-021` and, for a shard the later snapshot enumerates, the shards of the earlier one +its extent draws from. An integrator sizes a migration before deciding to run it, which is the +reason `TOPO-212` makes the ownership delta a call rather than a product of installation. + +`LIN-032`. An implementation MUST NOT compute a lineage as a step of `TOPO-001`, as part of +installing a snapshot, or as a precondition of a routing call reading an installed snapshot. + +`LIN-033`. The entries of a lineage MUST be ordered as `TOPO-213` orders an ownership delta: first +the entries for shards the later snapshot enumerates, in the order `shards` gives for that snapshot +under `PLACE-031`, then the entries for shards only the earlier snapshot enumerates, in the order +`shards` gives for it. + +`LIN-034`. Computing a lineage MUST cost no more than one walk of each snapshot's shard enumeration +and MUST NOT evaluate a candidate ordering. Under `ring` the two enumerations are ascending token +orders under `RING-031`, so the correspondence is a merge of two ordered sequences, and an +implementation MUST NOT materialise either enumeration where its shard count is the figure +`PLACE-075` bounds a delta over. + +#### Plan construction over a lineage + +`LIN-041`. `plan` MUST derive its handoffs from the lineage rather than from the ownership delta +alone, one parent at a time. For each shard `s` the later snapshot enumerates, for each shard `p` +whose extent `s` draws from under `LIN-004`, and for each node `d` of the replica set of `s` under +the later snapshot that is not an entry of the replica set of `p` under the earlier snapshot, a plan +MUST carry one handoff moving `p`'s contents for `s` to `d`. A destination already holding a +parent's contents needs no handoff for that parent, and a destination holding one parent's contents +still needs a handoff for every other parent it does not hold. + +`LIN-042`. An implementation MUST NOT emit a handoff whose source and destination are the same node. +A handoff names the node that holds the contents and the node that is to hold them, and a node +cannot be both. + +`LIN-043`. A shard classified `fresh` has no parent and no contents to move, so a plan MUST NOT emit +a handoff for it. Ownership of a fresh shard is established by the epoch change alone. + +`LIN-044`. A plan MUST include a handoff for a shard whose extent grew even where its replica set is +unchanged and the ownership delta therefore reports no entry for it. `TOPO-211` answers which shards +changed owner, which is a different question from which shards must receive contents, and a plan +built over the delta alone omits the destination of every fold whose replica set happens not to +change. + +`LIN-045`. The source of the handoff `LIN-041` admits MUST be chosen as follows. Let the departing +replicas of `p` be the entries of `p`'s replica set under the earlier snapshot that are not entries +of `s`'s replica set under the later snapshot, in the order the earlier snapshot gives, and let `i` +be the position of `d` among the destinations that parent admits, in the order the later snapshot +gives. The source MUST be the departing replica at position `i` where there is one, and otherwise +the first entry of `p`'s replica set under the earlier snapshot. A replica that is giving the shard +up is drained in preference to one that is keeping it, and a plan is a pure function of the two +snapshots and the policy under `MOVE-221`, so the choice MUST be stated rather than left to an +implementation. + ### Shard ownership handoff A handoff moves ownership of one shard from a source node to a destination node. The library @@ -3729,10 +3896,13 @@ routing conditions and does not govern these. ### Migration conditions -`ERR-050`. `planRefused` MUST be raised by `plan` under `MOVE-081`, `MOVE-251`, and `RATE-021`, and -by `rebase` under `MOVE-095`. It MUST carry a `cause` from the closed set `incomparableShards`, -`epochNotAdvancing`, `strategyUnsupported`, `destinationOutsidePlacementSet`, `policyInvalid`, and -`topologyMismatch`. +`ERR-050`. `planRefused` MUST be raised by `plan` under `MOVE-081`, `MOVE-251`, `RATE-021`, +`LIN-013`, `LIN-015`, and `LIN-022`, and by `rebase` under `MOVE-095`. It MUST carry a `cause` from +the closed set `incomparableShards`, `epochNotAdvancing`, `strategyUnsupported`, +`destinationOutsidePlacementSet`, `policyInvalid`, `topologyMismatch`, `unalignedLineage`, and +`lineageUnsupported`. The set is closed against a binding under `ERR-062`, which forbids a leaf of +`ERR-010` that this document does not state; a cause is not a leaf, and the two this batch adds are +raised by `plan` alone. `ERR-051`. `quiesced` MUST be raised for a write to a shard between the success of `quiesce` and the return of `commitCutover`, under `MOVE-311`. It MUST be reported as retryable at both the source and @@ -3915,6 +4085,7 @@ implementation MAY carry further members beyond those named. | `health.transition` | a health state changes | `info` | `node`, `from`, `to`, `trigger` | | `health.ejection_refused` | the ceiling refuses a transition | `warning` | `node`, `ejected`, `setSize` | | `fencing.refused` | a recipient refuses a request | `warning` | `relation`, `ownership`, `currentOwner`, `token` | +| `migration.lineage` | `LIN-031` is called | `info` | `unchanged`, `moved`, `divided`, `merged`, `fresh`, `split`, `folded`, `vacated` | | `migration.planned` | `plan` returns a plan | `info` | `handoffCount`, `policy` | | `migration.state_changed` | a handoff changes state | `info` | `handoff`, `shard`, `from`, `to`, `trigger` | | `migration.cutover_committed` | a record is committed | `info` | `shard`, `source`, `destination`, `windowMillis` | diff --git a/docs/design/30-conformance.md b/docs/design/30-conformance.md index 3ecc6a4..de575f0 100644 --- a/docs/design/30-conformance.md +++ b/docs/design/30-conformance.md @@ -751,6 +751,30 @@ them, `MOVE-151` through `MOVE-238`, because a state sequence is an output. `MOV `MOVE-103` are the same case: a rebase is a classification of each handoff against a snapshot, and both the classification and the state it leaves are outputs. +### Shard lineage + +`LIN-021` through `LIN-045` are classifications and plans, which are outputs, so +`vectors/migration/lineage.json` and `vectors/migration/plan-construction.json` carry them. Four of +the `LIN` requirements have no executable test, for reasons this section's other entries already +give. + +`LIN-005` and `LIN-023` constrain how a lineage is produced rather than what it is: the first +forbids reading health, a clock, or a handoff state, and the second forbids the enumeration order +changing a class. An implementation that departs from either produces a different classification +wherever the departure reaches one, and fails a lineage case there. + +`LIN-031` and `LIN-032` are the shape of an operation and a rule about where it is not called, +which is the case `Surface exposure` and `Concurrency and visibility` describe. `LIN-034` bounds +what a lineage costs, which is the case `Placement cost` describes. + +`LIN-015` binds a registered strategy outside the core set. The suite carries core-set documents, so +no data file can present one, and the same is true of every requirement about a registered strategy. + +`LIN-043` states that a plan emits no handoff for a shard with no parent. A fresh extent arises only +where a later snapshot admits routing keys that the earlier one matched to no shard, which is the +`directory` no-match of `DIR-010`, and `LIN-013` refuses a `directory` pair whose shard sets differ +until directory extents are defined. The case arrives with them. + ### Configuration and security `CFG-001` through `CFG-007`, `CFG-060` through `CFG-062`, and `CFG-064` govern how settings are diff --git a/docs/design/adr/0022-range-split-lineage.md b/docs/design/adr/0022-range-split-lineage.md index 08f877c..a88ce1d 100644 --- a/docs/design/adr/0022-range-split-lineage.md +++ b/docs/design/adr/0022-range-split-lineage.md @@ -7,6 +7,15 @@ The whole `SPLIT` prefix is withdrawn with the `range` strategy, which was the o the decomposition into local steps, and the `splitLocal` and `mergeLocal` hooks leave the design with it. The reasoning below is what a later minor version reads before adding the kind back. +Amended on 2026-09-19 by [`0086`](0086-shard-lineage-derived-from-extent.md). The lineage returns in +a different shape, under the `LIN` prefix and fresh identifiers, derived from each strategy's own +keyspace geometry rather than from authored range bounds. The `SPLIT` prefix stays withdrawn and +admits no further identifier. What this record got right and `0086` keeps: a lineage relates extents +rather than identifiers, a change in flight is two epochs and nothing else, an unaligned boundary +move is refused and republished as a division epoch followed by a fold epoch, and explicit lineage +members do not belong in the document. What `0086` does not take: the `range` strategy, the authored +bounds, and the rule that a division which has already succeeded cannot be undone. + ## Context The `range` strategy names shards by `shardId` and bounds them by a half-open interval. A split diff --git a/docs/design/adr/0054-range-strategy-withdrawal.md b/docs/design/adr/0054-range-strategy-withdrawal.md index d4ec059..0caa4f7 100644 --- a/docs/design/adr/0054-range-strategy-withdrawal.md +++ b/docs/design/adr/0054-range-strategy-withdrawal.md @@ -2,6 +2,14 @@ Status: accepted. Date: 2026-09-17. +Amended on 2026-09-19 by [`0086`](0086-shard-lineage-derived-from-extent.md). One sentence of the +Consequences below no longer holds: that a token addition under `ring` divides a token range as a +consequence of placement, which `SPLIT-001` already distinguished from a split. The distinction is +real about who decides and false about what the coordinator has to do, and its absence left `ring` +with no way to name the parent of a shard that appeared. `0086` carries that behaviour now, under +the `LIN` prefix. The withdrawal of the `range` strategy, of the `RANGE` and `SPLIT` prefixes, and +of `PROP-023`, `PROP-026`, and `CFG-063` is untouched, and `range` does not return. + ## Context [`0002`](0002-placement-strategy-set.md) shipped five strategy kinds, and justified `range` as the diff --git a/docs/design/adr/0086-shard-lineage-derived-from-extent.md b/docs/design/adr/0086-shard-lineage-derived-from-extent.md new file mode 100644 index 0000000..1ebbe98 --- /dev/null +++ b/docs/design/adr/0086-shard-lineage-derived-from-extent.md @@ -0,0 +1,104 @@ +# 0086. Shard lineage derived from extent + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0054`](0054-range-strategy-withdrawal.md) withdrew the `range` strategy and with it the whole +`SPLIT` prefix, on the ground that `SPLIT-001` admitted a split only under `range`. One sentence of +its Consequences does not hold. It reads that a token addition under `ring` divides a token range as +a consequence of placement, which `SPLIT-001` already distinguished from a split. The distinction is +real about who decides and false about what a coordinator has to do. A division that is a +consequence of placement still moves contents from one extent into another, and the machinery that +named the parent went out with the prefix. + +What that costs is visible in the port. Two `ring` snapshots at `factor` 2, with nodes `a` at tokens +`...1000` and `...5000` and `b` at `...3000` and `...7000`, gain a node `c` at the single token +`...2000`. The later snapshot enumerates a shard `...2000` that the earlier one does not, and the +contents of that shard are held by the replicas of `...3000` under the earlier snapshot. The +ownership delta reports that `...2000` appeared and that two nodes gained it. It cannot report where +the contents are, because `...3000` did not change owner and therefore has no delta entry at all. +The plan the port builds from that delta emits two handoffs whose source and destination are the +same node, and names neither node that holds the data. + +The reverse case is worse for being quieter. Removing `c` folds `...2000` back into `...3000`, whose +replica set is unchanged, so the shard that must receive the contents appears in no delta entry and +receives no handoff. + +This is reachable whenever a node joins or leaves a `ring` topology, which is the central operation +of the storage walkthrough in [`../00-overview.md`](../00-overview.md). It is not a missing feature +so much as a hole that split and merge are the names for. + +`TOPO-211` answers which shards changed owner. Nothing in the design answers which shards changed +contents, and the two questions have different answers exactly when an epoch changes which shards +exist. + +## Decision + +A **lineage** is added: the correspondence between the extents of two snapshots, where the extent of +a shard is the set of routing keys `shardOf` maps to it. Lineage relates extents and never +identifiers, which is [`0022`](0022-range-split-lineage.md)'s central insight carried forward +without the machinery that was specific to `range`. + +The lineage is derived from the topology document. No document member records it, no new strategy +kind is introduced, `formatVersion` stays at 1.0, and the JSON Schema is untouched. `0022`'s +Alternatives section rejected explicit lineage members on three grounds that all still hold: they +duplicate what the bounds, or here the tokens and the matchers, already carry; a hand-edited +document would omit them; and two members that can disagree need a rule for which one wins. + +The new requirements take the `LIN` prefix, in the Topology change and rebalancing section, on the +`migration` surface. `SPLIT` stays withdrawn and admits no further identifier, under +[`0053`](0053-requirement-withdrawal-convention.md). + +Lineage sits beside the ownership delta rather than inside it. `TOPO-211`, `TOPO-213`, and the +`ShardChange` shape are unchanged, and no `TOPO-*` requirement moves surface. + +## Consequences + +An epoch that divides or folds an extent is now plannable. A handoff for a divided extent names a +source drawn from the parent's replica set, and a fold produces a handoff for the shard that +receives the contents even where its replica set did not change, which `LIN-044` states because a +plan built over the delta alone omits it. + +The property the whole change rests on is that where two snapshots enumerate the same shard set the +lineage is the identity. Under every strategy the shard identifier of `PLACE-031` renders the +geometry that fixes the extent, so an equal shard set is an equal extent set. Every pair the suite +carried before this change is such a pair, so the regenerated tree is additions only, and a changed +expected value anywhere would mean the identity case had leaked. + +Lineage is kept out of the `routing` surface deliberately. Adding a parent to `ShardChange` would +put extent arithmetic at the `core` level, which [`../30-conformance.md`](../30-conformance.md) +says a port cannot decline, and would oblige every future port to implement ring interval +containment and directory precedence before declaring anything at all, for behaviour an integrator +routing tenant identifiers never uses. + +`range` does not return. Nothing here reinstates an ordered keyspace, a scan, or the `RANGE` prefix, +and a further strategy kind remains the minor format version that +[`../99-roadmap.md`](../99-roadmap.md) prices. + +## Alternatives + +Reinstating a range-like ordered strategy. Rejected. It costs a minor format version, a fifth entry +in a closed set, a fresh prefix of some forty identifiers, and the `assignment` default trap +[`0043`](0043-assignment-mode-defaults.md) chose to state loudly rather than remove, and at the end +of it split would be available only under the one kind that none of the three validated use cases +reaches. The `ring` hole would still be open. + +Making `slot` splittable by doubling `slotCount`. Rejected, and recorded here because the arithmetic +is more encouraging than the conclusion and should not be rediscovered. Writing a key hash as +`kn + i`, its remainder modulo `2n` is `i` where `k` is even and `i + n` where `k` is odd, so +doubling sends every key of slot `i` to slot `i` or slot `i + n` and nowhere else, for any `n`, with +no power-of-two requirement and no bitmask. A doubling is therefore a clean binary refinement of +every slot at once and a halving is its fold. It is still not worth building: under `slot` the +extent of a shard never changes while `slotCount` is held equal by `TOPO-231`, the delta and the +plan already work, and doubling addresses only a `slotCount` chosen too small, which an authority +avoids by choosing generously. + +A lineage layer above the strategies, with parent and child recorded per shard. Rejected for +`0022`'s reasons, restated above. + +Deriving a lineage by sampling routing keys and observing which shard each falls in. Rejected. It is +unsound at any sample size, and it would make a lineage depend on something outside the two +documents, which `LIN-005` forbids for the same reason `TOPO-221` forbids it of the delta. diff --git a/docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md b/docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md new file mode 100644 index 0000000..2133399 --- /dev/null +++ b/docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md @@ -0,0 +1,86 @@ +# 0087. Lineage capability derived rather than declared + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0086`](0086-shard-lineage-derived-from-extent.md) adds a lineage. A question it does not settle is +how an implementation knows whether a lineage is available for a given pair of snapshots, and the +obvious answer is wrong in a way worth recording. + +The obvious answer is a capability method on the placement extension point, defaulted to answer that +no lineage is offered. It reads as though it follows `MOVE-261`, and it does not. `MOVE-241` derives +support for orchestrated migration from observable geometry: a strategy supports it exactly when +`shards()` is non-empty and `shardOf` returns a shard identifier rather than the routing key. +`MOVE-261` binds a registered strategy to that same geometric rule and forbids inferring support +from the strategy's name. The design derives capability; a defaulted capability method declares it. + +The difference is not stylistic. [`../30-conformance.md`](../30-conformance.md) says a port declares +the levels it reaches, not a percentage. A port that declared the `migration` level while answering +that it offers no lineage for `ring` would be indistinguishable from a port that had simply not done +the work, and the configuration that produces is exactly the one that emits the self-handoffs +`0086` describes. + +Underneath the mistake were two different questions fused into one flag. + +## Decision + +The two questions are separated, and each is answered in its own way. + +Whether a **strategy** has a lineage is derived and never declared. A strategy has one exactly when +it supports orchestrated migration under `MOVE-241` and its shard extents are determined by the +topology document. `ring`, `slot`, and `directory` have one; `rendezvous` does not, because it +enumerates no shard. There is no opt-out, because the extents exist whether or not an implementation +has computed them. Lineage support is therefore not a property of the design at all, it is a +property of an implementation's completeness, and completeness is what the level system exists to +police. + +Whether the **integrator's storage** can divide a shard in place is declared, in +`HookDeclaration`. That is a statement about a system the library cannot observe, and +[`0085`](0085-hook-declarations-and-refused-aborts.md) is the precedent: `supportsRollback` and +`supportsVerify` are read as the integrator's statement about their own system, and the library +refuses rather than calling a hook that was never implemented. Nothing about it is a statement about +a port's completeness, so it does not collide with the level system. + +A registered strategy outside the core set is the one case where the library cannot derive the +answer, and it supplies its lineage rather than declaring a capability. `LIN-015` therefore forbids +deriving a registered strategy's extents by sampling and forbids inferring a lineage from the +strategy's name, and where a registered strategy supplies none and the two snapshots enumerate +different shard sets, the plan is refused with the cause `lineageUnsupported`. The method survives +but its meaning changes: it does not say "I support lineage", it says "here is my lineage", and its +absence has a stated, observable consequence rather than a silent one. + +## Consequences + +Per-strategy scoping does not need a new mechanism, because the suite already has one. The strategy +surface axis of [`../30-conformance.md`](../30-conformance.md) selects vector files on the surfaces +a port exposes, and `run_suite.py --strategy` already drives it. A port that exposes `rendezvous` +and `directory` and declares `migration` implements directory lineage and no ring lineage, and that +is expressible today. + +Staging within this repository is done with a stated refusal rather than a silent absence. +`LIN-013` refuses a `directory` pair whose shard sets differ, with the cause `unalignedLineage`, +until directory extents are defined; a later change restates that requirement and removes the +refusal, keeping its identifier under [`0053`](0053-requirement-withdrawal-convention.md). A refusal +is a conforming, testable behaviour that a vector pins, so no port can be green on a hole. + +This improves `directory` immediately rather than deferring it. A prefix refinement today produces +the same silent self-handoff as a ring token addition. Under the staged refusal it produces a +refused plan naming a cause, which is a correct answer in place of a wrong one. + +## Alternatives + +A defaulted capability method meaning "I offer no lineage". Rejected above: it makes an incomplete +port indistinguishable from a complete one, against the rule that a port declares levels rather than +a percentage. + +A per-strategy capability flag in the conformance declaration. Rejected. It duplicates the strategy +surface axis that already exists, and it would let a port declare `migration` and `ring` together +while opting out of the one thing that makes the pair meaningful. + +Recording directory lineage under "Requirements without an executable test" so it could ship before +its vectors. Rejected. A directory extent is decidable over a prefix trie, so it is testable, and +recording something as untestable to avoid writing its vectors is the one dishonest move available +here. diff --git a/docs/design/adr/0088-lineage-inside-the-migration-surface.md b/docs/design/adr/0088-lineage-inside-the-migration-surface.md new file mode 100644 index 0000000..b0fcc58 --- /dev/null +++ b/docs/design/adr/0088-lineage-inside-the-migration-surface.md @@ -0,0 +1,54 @@ +# 0088. Lineage inside the migration surface + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0086`](0086-shard-lineage-derived-from-extent.md) adds the `LIN` prefix. Where its requirements +are tested, and which implementations they bind, is a separate judgement, because +[`0058`](0058-conformance-surfaces.md) makes a surface a unit an implementation exposes whole or not +at all and [`0052`](0052-conformance-level-partition.md) fixes the partition of the suite into +levels. + +Three placements were available: a new `lineage` surface with a level of its own, the existing +`migration` surface and level, or the `routing` surface alongside the ownership delta. + +## Decision + +Lineage belongs to the existing `migration` surface and is tested at the existing `migration` level. +No surface is added and the level partition does not move. + +## Consequences + +A port that declines the `migration` surface implements no lineage and is bound by no `LIN` +requirement, which is the same answer `0058` already gives for every other part of the handoff +coordinator. + +A port that exposes `migration` implements lineage for every strategy surface it also exposes. There +is no configuration in which a port exposes the handoff coordinator over `ring` and has no answer +for a token addition, because that configuration is the one that emits the self-handoffs `0086` +describes. + +The level partition is untouched, symmetrically with `0054`'s own argument that removing artefacts +from a level does not move the partition. `migration` already requires `failover`, and it gains +vector files rather than a new requires edge. + +The Java port declares every level, so it pays for lineage at `migration` and for the `TOPO-213` +repair at `core`. No other port exists. A future port declining `migration` pays nothing, and a +future port exposing only `rendezvous` pays nothing beyond the `core` repair, because `rendezvous` +enumerates no shard and `MOVE-241` already excludes it. + +## Alternatives + +A `lineage` surface of its own. Rejected because it would let a port declare `migration` without it, +which is the broken configuration named above. + +Lineage on the `routing` surface, as a parent member of `ShardChange`. Rejected in `0086`'s +Consequences: it would put ring interval containment and directory precedence at the `core` level, +which a port cannot decline, for behaviour an integrator routing tenant identifiers never uses. + +A new conformance level. Rejected. A level tests one surface, `migration` is that surface, and a +second level over the same surface would let a port claim the coordinator while declining the part +that makes it correct. diff --git a/docs/design/adr/0091-lineage-classification-as-a-signal.md b/docs/design/adr/0091-lineage-classification-as-a-signal.md new file mode 100644 index 0000000..02d0245 --- /dev/null +++ b/docs/design/adr/0091-lineage-classification-as-a-signal.md @@ -0,0 +1,65 @@ +# 0091. Lineage classification as a signal + +Date: 2026-09-19 + +Status: accepted + +## Context + +The motivation for [`0086`](0086-shard-lineage-derived-from-extent.md) was that splitting and +merging should be the ordinary way to change capacity, in preference to republishing a topology that +redistributes everything. That raises a question the lineage does not answer by itself: whether the +library should do anything about an authority that reshards wholesale when a split would have done. + +Three positions were available. Refuse the plan where the lineage shows a cheaper change existed. +Emit a warning. Do neither, and let the classification be reported. + +The library authors no topology document, assigns no epoch, and edits nothing. +[`0022`](0022-range-split-lineage.md) was firm that the decision to divide belongs to the authority, +and nothing in `0086` changes that. + +## Decision + +The library refuses `unaligned` and refuses nothing else. + +`LIN-022` refuses a plan over a lineage that classifies any shard `unaligned`, because a boundary +that moves without either dividing or folding an extent whole has no correspondence an +implementation can name. That is a correctness refusal over an ambiguous input, not a policy one, +and the requirement states the remedy, which is `0022`'s and still the right one: publish the change +as two epochs, the first dividing every extent the change crosses and the second folding the pieces +into their destinations. + +The classification is reported through the `migration.lineage` event of `OBS-020`, which carries the +count per class. The planner needs the classification to choose a source, so reporting it costs +nothing. + +No refusal and no warning is raised for a change that could have been expressed as a division. The +preference is carried by documentation, which states when a division is the right shape, when a +wholesale republication is, and what each costs. + +## Consequences + +An operator reads from `migration.lineage` whether an epoch was a refinement or a redistribution, +and decides what to do about it. The library states the facts and takes no position. + +The strongest expression of the preference is not a rule at all. Before `0086` an authority that +added one node to a ring got a plan of self-handoffs, and an authority that republished everything +got the same, so nobody preferred anything because neither worked. Making a division correct and +cheap is most of what "preferred" can honestly mean here. + +An authority that reshards wholesale where a division would have served pays for it in movement and +is told nothing by the library. That is the accepted cost of not guessing. + +## Alternatives + +Refusing a plan where the lineage shows a division was available. Rejected. The library cannot +distinguish a deliberate rebalance, a seed rotation, or a failure domain re-layout from a lazy +reshard, and a false refusal blocks an epoch the authority has already published to routing, which +is a worse failure than the one it would prevent. + +A warning event for the same condition. Rejected for the same reason at lower value: it needs a +threshold for what counts as wholesale, which is a judgement the library has no basis to make, and +an operator who cannot act on it learns to ignore it. + +A configurable policy selecting among the three. Rejected. It is a knob for a policy nobody has +asked for, and it would oblige every port to implement all three positions. diff --git a/ports/java/USER-GUIDE.md b/ports/java/USER-GUIDE.md index 178ca8c..22a2954 100644 --- a/ports/java/USER-GUIDE.md +++ b/ports/java/USER-GUIDE.md @@ -533,13 +533,69 @@ topology before retrying, and never a reason to install that epoch. [`10-specification.md`](../../docs/design/10-specification.md#caller-behaviour-when-fenced) states the walk. -## Resharding +## Resharding and shard lineage -The library has no split operation and no merge operation. -[`adr/0054`](../../docs/design/adr/0054-range-strategy-withdrawal.md) withdrew the `range` strategy -and the split prefix whole. Resharding takes a different shape: an authority publishes a topology at -a higher epoch with a different node set or a different replication factor, ownership of the -existing shards moves, and the delta between the two topologies names exactly what moved. +An authority changes a topology by publishing a whole document at a higher epoch. What that change +does to the shards is the thing worth being deliberate about, and there are two shapes. + +A **reshard** keeps the shards and moves their ownership. The node set or the replication factor +changes, every shard keeps its identifier and the keys it holds, and the ownership delta names the +shards whose replica set differs. + +A **split** or a **merge** changes which shards exist. Adding a ring token divides the extent of one +shard into two; removing one folds two extents into one. The shard identifiers are not the same on +both sides, so the ownership delta, which joins on the identifier, has no answer for the ones that +appeared or vanished. The lineage is the second join, over the keys a shard holds, and it is what +tells a plan where a new shard's contents come from. + +The library derives the lineage from the topology document. Nothing in the document records it, and +no member has to be authored to get a split. + +### Which shape to reach for + +Prefer a split or a merge where the change is about capacity for part of the keyspace. + +| | Split or merge | Reshard | +|---|---|---| +| What changes | which shards exist, and the keys each holds | who owns the existing shards | +| What moves | the contents of the extents that divided or folded | the contents of every shard whose replica set changed | +| Typical cause | one shard outgrew a node, or two are small enough to combine | a node joined or left, or the replication factor changed | +| Cost | proportional to the extent that moved | proportional to what the strategy reassigns | +| Reversible | yes, by folding back or dividing again | yes, by publishing the earlier shape | + +A split moves less because it disturbs less: the keys outside the divided extent do not change +shard, so nothing about them moves. A reshard that redistributes the whole keyspace to add capacity +moves data that was already where it belonged. Where both would serve, the split is the cheaper +change, and it is the one to publish. + +A reshard is the right shape when what changed is the cluster rather than the keyspace. Adding a +node to a `ring` topology under derived tokens is a reshard and a split at once, because the node's +tokens divide the extents they land in, and the library plans it as one change. + +Under `slot` the shard count is fixed by `slotCount`, which `TOPO-231` holds equal across a +comparable pair, so every change is a reshard. Choose `slotCount` generously at the outset: it is +the one decision here that a later epoch cannot revisit. + +The library does not enforce the preference. It refuses a change whose boundaries neither divide nor +fold an extent whole, because that has no lineage to name, and it reports the classification through +the `migration.lineage` event so an operator can see which shape an epoch took. It refuses nothing +else, and it never declines a reshard on the ground that a split would have been cheaper, because it +cannot tell a deliberate rebalance from a lazy one. +[`adr/0091`](../../docs/design/adr/0091-lineage-classification-as-a-signal.md) argues that. + +### The unaligned change + +A change that moves a boundary without either dividing or folding an extent whole is refused with +`PlanRefusedException` and the cause `unalignedLineage`. Removing a ring token and adding a +different one inside the same extent in one epoch is the common way to reach it. + +Publish it as two epochs instead: the first divides every extent the change crosses, the second +folds the pieces into their destinations. Each is plannable on its own, and each leaves a topology +that routes correctly if the second is delayed. + +Under `directory` a pair whose entry sets differ is refused for now with the same cause, until +directory extents are defined. A directory topology whose entries are unchanged plans as it always +did. ### A new epoch @@ -577,9 +633,11 @@ A `ShardChange` carries the shard, the replica set before, the replica set after differences. The sets are the entries whose role is `REPLICA`, which is the achieved replica prefix rather than the whole preference list, so a node a shard merely falls back to has gained nothing. -The delta is computed on demand rather than at installation, and a shard the later snapshot does not -enumerate is absent from it. Two snapshots whose shard identity is not comparable refuse the -computation with `InvalidArgumentException` rather than reporting every shard as wholly changed. +The delta is computed on demand rather than at installation. Its entries come in two groups: first +the shards the later snapshot enumerates, in that snapshot's order, then the shards only the earlier +one enumerates, so a shard that vanished reports the nodes that lost it rather than going +unreported. Two snapshots whose shard identity is not comparable refuse the computation with +`InvalidArgumentException` rather than reporting every shard as wholly changed. Under `rendezvous` the delta is empty, because the strategy enumerates no shards. Reading the delta is enough where the integrator's store moves data by itself. Where each move has diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index 81492ec..d04c51c 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,17 +1,17 @@ sharder conformance report, Java port -suite revision 6203a8effcf62cfe7bd9416b38656d8c6c7bbae4e1885f065ffe8b7e83a688a9 +suite revision 3f56d0a573f1aaf7f364802e43a2bb0d7682eb08a8e7632eb89b86728c682070 hash 2 of 2 vector files, 34 of 34 cases, 0.01s - place 45 of 45 vector files, 291 of 291 cases, 0.11s - core 8 of 8 vector files, 135 of 135 cases, 8.25s - scale 2 of 2 vector files, 2 of 2 cases, 1.01s + place 45 of 45 vector files, 291 of 291 cases, 0.09s + core 8 of 8 vector files, 135 of 135 cases, 8.02s + scale 2 of 2 vector files, 2 of 2 cases, 0.90s failover 3 of 3 vector files, 67 of 67 cases, 0.00s readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s fencing 3 of 3 vector files, 8 of 8 cases, 0.00s - migration 3 of 3 vector files, 15 of 15 cases, 0.00s + migration 5 of 5 vector files, 25 of 25 cases, 0.01s - 598 cases, 0 failures - scale wall time 1.01s - scale wall time 1006 ms - peak resident size 616009728 bytes (587 MiB) + 608 cases, 0 failures + scale wall time 0.90s + scale wall time 899 ms + peak resident size 591892480 bytes (564 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java index 1144600..9debdf3 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java @@ -16,6 +16,7 @@ public final class Handoff { private final String id; private final String shard; + private final String sourceShard; private final NodeId source; private final NodeId destination; private final long fromEpoch; @@ -27,8 +28,14 @@ public final class Handoff { Handoff(String id, String shard, NodeId source, NodeId destination, long fromEpoch, long toEpoch) { + this(id, shard, shard, source, destination, fromEpoch, toEpoch); + } + + Handoff(String id, String shard, String sourceShard, NodeId source, NodeId destination, + long fromEpoch, long toEpoch) { this.id = id; this.shard = shard; + this.sourceShard = sourceShard; this.source = source; this.destination = destination; this.fromEpoch = fromEpoch; @@ -45,6 +52,18 @@ public String shard() { return shard; } + /** + * The shard whose contents this handoff moves, which {@code LIN-041} draws from the lineage. + * + *

It is the shard itself wherever the two snapshots enumerate the same shards. Where the + * later snapshot divided or folded an extent, the contents live under the parent's identifier + * at the source, so a hook that looked only at {@link #shard()} would search the source for a + * shard it does not hold. + */ + public String sourceShard() { + return sourceShard; + } + /** The node that owns the shard before the cutover. */ public NodeId source() { return source; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java index 70b2a35..0c24300 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java @@ -1,6 +1,7 @@ package com.codeheadsystems.sharder.core.internal.migrate; import com.codeheadsystems.sharder.NodeId; +import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; import com.codeheadsystems.sharder.migrate.HandoffState; @@ -94,8 +95,19 @@ public static Handoff handoffOf(String id, String shard, NodeId source, NodeId d } /** - * The plan between two snapshots: one handoff per shard that gained a node, under the - * ownership delta of {@code TOPO-211}. + * The plan between two snapshots, derived from the lineage under {@code LIN-041}. + * + *

One handoff is admitted for each shard of the later snapshot, each shard its extent draws + * from, and each destination that does not already hold that parent's contents. Where the two + * snapshots enumerate the same shards the lineage is the identity of {@code LIN-007} and every + * shard is its own parent, which is the ownership delta's answer and the case every topology + * the suite carried before the lineage falls into. + * + *

Building the plan over the delta alone is what {@code LIN-044} forbids. A shard that + * absorbed a folded extent keeps its replica set, so the delta reports no entry for it, and a + * destination that holds one parent but not the other would receive nothing. A shard divided + * out of a parent has no entry in the earlier snapshot at all, so the source would fall back to + * the destination and the plan would tell a node to copy from itself. * *

Plan construction is a pure function of the two snapshots and the policy, which is what * lets a restarted coordinator rebuild the same plan under {@code MOVE-221}. @@ -103,22 +115,36 @@ public static Handoff handoffOf(String id, String shard, NodeId source, NodeId d public static MigrationPlan of(PlacementEngine from, PlacementEngine to, long quiesceLeaseMarginMillis) { MigrationPlan plan = new MigrationPlan(quiesceLeaseMarginMillis, to.document().epoch()); - for (OwnershipDelta.ShardChange change : OwnershipDelta.between(from, to)) { - // A shard that gained a node and lost one is a move from that source to that - // destination; the pairing is by position, which the delta reports in identity order. - for (int entry = 0; entry < change.gained().size(); entry++) { - NodeId destination = change.gained().get(entry); - NodeId source = entry < change.lost().size() - ? change.lost().get(entry) - : change.before().isEmpty() ? destination : change.before().get(0); - // A handoff is named after the shard it moves, so a rebuilt plan names the same - // handoffs, which MOVE-221 rests on. - String id = "h-" + change.shard(); - while (plan.handoffs.containsKey(id)) { - id = id + "-" + entry; + Map> parents = ShardExtents.parents(from, to); + for (String shard : to.placement().shards()) { + List destinations = OwnershipDelta.replicas(to, shard); + for (String parent : parents.getOrDefault(shard, List.of(shard))) { + List held = OwnershipDelta.replicas(from, parent); + if (held.isEmpty()) { + // LIN-043: a shard with no parent has no contents to move. + continue; + } + List needing = new ArrayList<>(destinations); + needing.removeAll(held); + List departing = new ArrayList<>(held); + departing.removeAll(destinations); + for (int entry = 0; entry < needing.size(); entry++) { + // LIN-045: a replica giving the shard up is drained in preference to one + // keeping it, so a plan rebuilt from the same pair of snapshots names the same + // source for the same destination. + NodeId source = entry < departing.size() + ? departing.get(entry) + : held.get(0); + NodeId destination = needing.get(entry); + String id = "h-" + shard; + int suffix = 1; + while (plan.handoffs.containsKey(id)) { + id = "h-" + shard + "-" + suffix; + suffix++; + } + plan.handoffs.put(id, new Handoff(id, shard, parent, source, destination, + from.document().epoch(), to.document().epoch())); } - plan.handoffs.put(id, new Handoff(id, change.shard(), source, destination, - from.document().epoch(), to.document().epoch())); } } return plan; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java index 2a4120e..4304fbc 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java @@ -60,6 +60,10 @@ public static List eventsOf(String surface) { /** The events of the {@code migration} surface. */ public static final List MIGRATION_EVENTS = List.of( + // LIN-031: the count per class, which is where an operator reads whether an + // epoch was a refinement or a redistribution. Under 0091 nothing acts on it. + event("migration.lineage", "info", "unchanged", "moved", "divided", "merged", + "fresh", "split", "folded", "vacated"), event("migration.planned", "info", "handoffCount", "policy"), event("migration.state_changed", "info", "handoff", "shard", "from", "to", "trigger"), event("migration.cutover_committed", "info", "shard", "source", "destination", diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java new file mode 100644 index 0000000..0e517d9 --- /dev/null +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java @@ -0,0 +1,273 @@ +package com.codeheadsystems.sharder.core.internal.placement; + +import com.codeheadsystems.sharder.core.internal.hash.U64; +import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; +import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Shard extents and the lineage over them, under {@code LIN-001} through {@code LIN-034}. + * + *

The extent of a shard is the set of routing keys {@code shardOf} maps to it, determined by the + * document alone under {@code LIN-001}. Under {@code ring} an extent is the half-open token + * interval of {@code RING-020}, which wraps at zero, so containment is interval arithmetic over + * unsigned 64-bit bounds and every comparison here goes through {@link U64}. A signed comparison + * would read an extent near the top of the space as empty, which is why this class sits in the + * package {@code verifyUnsignedComparisons} covers rather than beside the coordinator. + * + *

Under {@code slot} {@code TOPO-231} holds {@code slotCount} equal, and under {@code directory} + * {@code LIN-013} refuses a pair whose shard sets differ, so both are the identity lineage of + * {@code LIN-007} and neither reaches the interval arithmetic. + */ +public final class ShardExtents { + + /** The class of one shard under {@code LIN-021}. */ + public enum Lineage { + UNCHANGED, MOVED, DIVIDED, MERGED, FRESH, SPLIT, FOLDED, VACATED + } + + /** One shard's class and the shards its extent draws from. */ + public record Entry(String shard, Lineage lineage, List parents) { + } + + /** A lineage that cannot be computed, carrying the cause {@code ERR-050} reports. */ + public static final class Refused extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final String cause; + + Refused(String cause, String detail) { + super(cause + ": " + detail); + this.cause = cause; + } + + public String cause() { + return cause; + } + } + + /** + * One inclusive segment {@code [lo, hi]} of the hash space, with unsigned 64-bit bounds. + * + *

Bounds are inclusive rather than half-open because the space ends at an unsigned value a + * {@code long} cannot hold one past, and an interval that wraps past zero is two segments, so + * every segment here has {@code lo} at or below {@code hi} and no arithmetic reasons about a + * wrap. Nothing measures a width: a width of the whole space overflows, so containment is + * decided by subtraction instead. + */ + private record Segment(long lo, long hi) { + } + + private ShardExtents() { + } + + /** The keys {@code h} with {@code lowExclusive < h <= highInclusive}, as segments. */ + private static List segments(long lowExclusive, long highInclusive) { + // An equality of two 64-bit values compiles to LCMP, which the placement path forbids, so + // the comparison goes through U64 like every other one here. + if (U64.compare(lowExclusive, highInclusive) == 0) { + // One token owns the whole space, from zero to the greatest unsigned value. + return List.of(new Segment(0L, -1L)); + } + long lo = lowExclusive + 1; + if (U64.compare(lo, highInclusive) <= 0) { + return List.of(new Segment(lo, highInclusive)); + } + // The interval wraps past zero, so it is the head of the space and its tail. + return List.of(new Segment(0L, highInclusive), new Segment(lo, -1L)); + } + + /** The extent of each shard the ring names, under {@code LIN-011}. */ + private static Map> ringExtents(List shards) { + Map> out = new LinkedHashMap<>(); + int count = shards.size(); + for (int index = 0; index < count; index++) { + long token = U64.parseHex(shards.get(index)); + long predecessor = count == 1 + ? token + : U64.parseHex(shards.get((index - 1 + count) % count)); + out.put(shards.get(index), segments(predecessor, token)); + } + return out; + } + + private static Optional overlap(Segment left, Segment right) { + long lo = U64.max(left.lo(), right.lo()); + long hi = U64.min(left.hi(), right.hi()); + return U64.compare(lo, hi) <= 0 ? Optional.of(new Segment(lo, hi)) : Optional.empty(); + } + + private static boolean meets(List first, List second) { + for (Segment left : first) { + for (Segment right : second) { + if (overlap(left, right).isPresent()) { + return true; + } + } + } + return false; + } + + /** What remains of {@code inner} once every segment of {@code outer} is taken out of it. */ + private static List subtract(List inner, List outer) { + List remaining = new ArrayList<>(inner); + for (Segment cut : outer) { + List next = new ArrayList<>(); + for (Segment piece : remaining) { + Optional shared = overlap(piece, cut); + if (shared.isEmpty()) { + next.add(piece); + continue; + } + Segment taken = shared.get(); + if (U64.compare(piece.lo(), taken.lo()) < 0) { + next.add(new Segment(piece.lo(), taken.lo() - 1)); + } + if (U64.compare(taken.hi(), piece.hi()) < 0) { + next.add(new Segment(taken.hi() + 1, piece.hi())); + } + } + remaining = next; + } + return remaining; + } + + private static boolean contains(List outer, List inner) { + return subtract(inner, outer).isEmpty(); + } + + private static boolean equal(List first, List second) { + return contains(first, second) && contains(second, first); + } + + /** + * For each shard the later snapshot names, the shards of the earlier one its extent draws from, + * under {@code LIN-004}. + */ + public static Map> parents(PlacementEngine before, PlacementEngine after) { + List earlier = before.placement().shards(); + List later = after.placement().shards(); + if (earlier.equals(later) || !"ring".equals(after.document().strategy().kind())) { + Map> identity = new LinkedHashMap<>(); + later.forEach(shard -> identity.put(shard, List.of(shard))); + return identity; + } + Map> earlierExtents = ringExtents(earlier); + Map> laterExtents = ringExtents(later); + Map> out = new LinkedHashMap<>(); + for (String shard : later) { + List drawn = new ArrayList<>(); + for (String candidate : earlier) { + if (meets(earlierExtents.get(candidate), laterExtents.get(shard))) { + drawn.add(candidate); + } + } + out.put(shard, List.copyOf(drawn)); + } + return out; + } + + /** + * The lineage of two snapshots, in the order {@code LIN-033} fixes. + * + *

{@code replicaSet} answers a shard's replica set under one snapshot, which separates + * {@code unchanged} from {@code moved} and is the only thing here that reads placement. + */ + public static List classify(PlacementEngine before, PlacementEngine after, + ReplicaSet replicaSet) { + Optional differing = OwnershipDelta.incomparable(before.document(), + after.document()); + if (differing.isPresent()) { + throw new Refused("incomparableShards", differing.get()); + } + String kind = after.document().strategy().kind(); + if ("rendezvous".equals(kind)) { + throw new Refused("strategyUnsupported", "rendezvous enumerates no shard"); + } + List earlier = before.placement().shards(); + List later = after.placement().shards(); + if (!"ring".equals(kind)) { + if (!earlier.equals(later)) { + throw new Refused("unalignedLineage", + "the two snapshots enumerate different shards"); + } + return identity(later, before, after, replicaSet); + } + if (earlier.equals(later)) { + return identity(later, before, after, replicaSet); + } + + Map> earlierExtents = ringExtents(earlier); + Map> laterExtents = ringExtents(later); + Set enumeratedLater = new LinkedHashSet<>(later); + List entries = new ArrayList<>(); + for (String shard : later) { + List mine = laterExtents.get(shard); + List drawn = new ArrayList<>(); + for (String candidate : earlier) { + List theirs = earlierExtents.get(candidate); + if (!meets(theirs, mine)) { + continue; + } + if (!contains(theirs, mine) && !contains(mine, theirs)) { + throw new Refused("unalignedLineage", shard); + } + drawn.add(candidate); + } + if (drawn.isEmpty()) { + entries.add(new Entry(shard, Lineage.FRESH, List.of())); + } else if (drawn.size() > 1) { + entries.add(new Entry(shard, Lineage.MERGED, List.copyOf(drawn))); + } else if (equal(earlierExtents.get(drawn.get(0)), mine)) { + entries.add(new Entry(shard, moved(shard, before, after, replicaSet), + List.copyOf(drawn))); + } else { + entries.add(new Entry(shard, Lineage.DIVIDED, List.copyOf(drawn))); + } + } + for (String shard : earlier) { + if (enumeratedLater.contains(shard)) { + continue; + } + List mine = earlierExtents.get(shard); + List children = new ArrayList<>(); + for (String candidate : later) { + if (meets(laterExtents.get(candidate), mine)) { + children.add(candidate); + } + } + Lineage lineage = children.isEmpty() ? Lineage.VACATED + : children.size() > 1 ? Lineage.SPLIT : Lineage.FOLDED; + entries.add(new Entry(shard, lineage, List.copyOf(children))); + } + return List.copyOf(entries); + } + + private static List identity(List shards, PlacementEngine before, + PlacementEngine after, ReplicaSet replicaSet) { + List entries = new ArrayList<>(); + for (String shard : shards) { + entries.add(new Entry(shard, moved(shard, before, after, replicaSet), List.of(shard))); + } + return List.copyOf(entries); + } + + private static Lineage moved(String shard, PlacementEngine before, PlacementEngine after, + ReplicaSet replicaSet) { + return replicaSet.of(before, shard).equals(replicaSet.of(after, shard)) + ? Lineage.UNCHANGED : Lineage.MOVED; + } + + /** A shard's replica set under one snapshot, which {@code TOPO-211} defines. */ + @FunctionalInterface + public interface ReplicaSet { + List of(PlacementEngine engine, String shard); + } +} diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java b/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java index ccc6d48..6cd1e6b 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java @@ -168,7 +168,7 @@ public List causes() { "retryBudget"); case PLAN_REFUSED -> List.of("incomparableShards", "epochNotAdvancing", "strategyUnsupported", "destinationOutsidePlacementSet", "policyInvalid", - "topologyMismatch"); + "topologyMismatch", "unalignedLineage", "lineageUnsupported"); case HANDOFF_FAILED -> List.of("unverified", "residue", "undetermined", "rollbackFailed"); default -> List.of(); diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java index e1ff8c9..fb1e348 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java @@ -143,6 +143,8 @@ private void runCase(String kind, JsonObject testCase, PlaceVectors place, case "observabilityInventory" -> core.observabilityInventory(testCase); case "publicationEvents" -> core.publicationEvents(testCase); case "ownershipDelta" -> core.ownershipDelta(testCase); + case "lineage" -> core.lineage(testCase); + case "planConstruction" -> core.planConstruction(testCase); case "propertyWitness" -> core.propertyWitness(testCase); case "scale" -> core.scale(testCase); case "readAffinity" -> core.readAffinity(testCase); diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java index 56843c8..1a134e4 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java @@ -1,6 +1,7 @@ package com.codeheadsystems.sharder.conformance; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.codeheadsystems.sharder.Digest; import com.codeheadsystems.sharder.NodeId; @@ -13,6 +14,9 @@ import com.codeheadsystems.sharder.core.internal.observe.Observability; import com.codeheadsystems.sharder.core.internal.observe.PublicationEvents; import com.codeheadsystems.sharder.core.internal.observe.SkewDetection; +import com.codeheadsystems.sharder.core.internal.migrate.Handoff; +import com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan; +import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; import com.codeheadsystems.sharder.error.ErrorCode; @@ -289,6 +293,67 @@ void ownershipDelta(JsonObject testCase) { } } + /** {@code lineage}: the classification of two snapshots' extents, under {@code LIN-021}. */ + void lineage(JsonObject testCase) { + PlacementEngine before = engine(testCase, "before"); + PlacementEngine after = engine(testCase, "after"); + JsonObject expect = testCase.object("expect"); + ShardExtents.ReplicaSet replicas = OwnershipDelta::replicas; + if (expect.find("lineageComputed").isPresent()) { + JsonObject condition = expect.object("condition"); + ErrorCode code = ErrorCode.ofName(condition.text("name")); + assertThat(code.code()).as("condition code").isEqualTo(condition.get("code").asInt()); + assertThatThrownBy(() -> ShardExtents.classify(before, after, replicas)) + .as("refusal").isInstanceOf(ShardExtents.Refused.class); + try { + ShardExtents.classify(before, after, replicas); + } catch (ShardExtents.Refused refusal) { + assertThat(refusal.cause()).as("cause").isEqualTo(condition.text("cause")); + assertThat(code.causes()).as("cause is in the closed set") + .contains(condition.text("cause")); + } + return; + } + List entries = ShardExtents.classify(before, after, replicas); + List expected = expect.array("lineage").elements(); + assertThat(entries).as("lineage").hasSize(expected.size()); + for (int index = 0; index < expected.size(); index++) { + JsonObject entry = expected.get(index).asObject(); + ShardExtents.Entry actual = entries.get(index); + assertThat(actual.shard()).as("lineage[%d].shard", index) + .isEqualTo(entry.text("shard")); + assertThat(actual.lineage().name().toLowerCase(java.util.Locale.ROOT)) + .as("lineage[%d].class", index).isEqualTo(entry.text("class")); + assertThat(actual.parents()).as("lineage[%d].parents", index) + .isEqualTo(entry.array("parents").texts()); + } + } + + /** {@code planConstruction}: how a plan derives a handoff from a lineage, {@code LIN-041}. */ + void planConstruction(JsonObject testCase) { + PlacementEngine before = engine(testCase, "before"); + PlacementEngine after = engine(testCase, "after"); + MigrationPlan plan = MigrationPlan.of(before, after, 0L); + List expected = testCase.object("expect").array("handoffs").elements(); + assertThat(plan.handoffs()).as("handoff count").hasSize(expected.size()); + for (int index = 0; index < expected.size(); index++) { + JsonObject entry = expected.get(index).asObject(); + Handoff handoff = plan.handoff(entry.text("id")); + assertThat(handoff).as("handoffs[%d] named %s", index, entry.text("id")).isNotNull(); + assertThat(handoff.shard()).as("handoffs[%d].shard", index) + .isEqualTo(entry.text("shard")); + assertThat(handoff.sourceShard()).as("handoffs[%d].sourceShard", index) + .isEqualTo(entry.text("sourceShard")); + assertThat(handoff.source().asText()).as("handoffs[%d].source", index) + .isEqualTo(entry.text("source")); + assertThat(handoff.destination().asText()).as("handoffs[%d].destination", index) + .isEqualTo(entry.text("destination")); + // LIN-042: a node cannot both hold the contents and be the node they move to. + assertThat(handoff.source()).as("handoffs[%d] source is not the destination", index) + .isNotEqualTo(handoff.destination()); + } + } + /** {@code propertyWitness}: a sampled bound over the sample {@code PROP-006} fixes. */ void propertyWitness(JsonObject testCase) { JsonObject expect = testCase.object("expect"); From 7747423ef3c26741f69b948d084c2b58648c6bc5 Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 08:24:47 -0700 Subject: [PATCH 3/7] Run the local step a division leaves on one node A division that moves no data still has work to do. A node that holds the parent under the earlier snapshot and the child under the later one has to divide its own copy so that what it holds matches the extent it now owns, and the plan the lineage produced named no such step. The keys were in the right place and the node's idea of which shard they belonged to was not. `MOVE-001` gains `dividing` and `MOVE-021` gains four transitions, which is the largest single edit in this work because that requirement says "exactly these transitions and no others". It lands after the lineage itself is settled and vector-backed rather than beside it. A local step names one node as both its source and its destination, which is the one place `LIN-042` does not apply, and runs `planned` to `dividing` to `complete`. There is no second party to prepare, quiesce, or cut over to, so the long sequence would be meaningless rather than merely wasteful. `LIN-057` orders it: a division before every handoff that draws from the divided parent, a fold after every handoff that draws into the folded shard, because a destination copying from a parent being divided underneath it would copy an extent that is changing. The step is reversible, and that is the point of 0089. `0022` had a local split that could not be undone, which is where `SPLIT-171` came from, and it was the single carve-out in `MOVE-233` that `0051` argued against and `0054` was glad to remove. Requiring the inverse hook keeps `MOVE-233` unconditional: a division is undone by `combine` and a fold by `divide`, so `dividing` reaches `aborting` like every other state that does and `MOVE-421` names it beside the other three. The cost lands on the integrator where it can be seen. Whether a storage can divide a copy in place is a property of that storage, so `HookDeclaration` gains `supportsLineage` and `LIN-053` refuses a plan that needs a local step without it, in preference to calling a hook that was never implemented. That is the shape 0085 established for `supportsRollback`. `HandoffContext` gains `sourceShardId`. It is the member doing the real work in an ordinary handoff too: a transfer hook for a divided child is told to move a shard the source does not hold, and without the parent in the context it cannot find the bytes. `MOVE-011` gains the failure kind `undivided`, for a copy that matches neither the parent's extent nor the child's. The four kinds it already carried each name a condition of a copy that moved between nodes, and this one never left the node it is on. 0090 records what is deliberately not done: a handoff whose shard a newer snapshot no longer enumerates is aborted rather than rewritten onto the children. That is what `MOVE-096` already says and what the port already does, and a transitive rebase would have to decide which child inherits a partial copy and whether a quiesce lease taken against the parent still binds. Relaxing a refusal later is cheap; withdrawing a clever rebase that proved wrong is not. Co-Authored-By: Claude Opus 5 --- conformance/coverage.json | 64 +++++++++---- conformance/declarations/java.json | 6 +- conformance/generator/generate_extra.py | 13 +-- conformance/generator/generate_scenarios.py | 50 ++++++++++- conformance/generator/property_definitions.py | 4 +- conformance/generator/sharder_ref/handoff.py | 20 ++++- conformance/generator/sharder_ref/lineage.py | 42 +++++++-- conformance/manifest.json | 71 ++++++++++++--- conformance/properties/properties.json | 4 +- .../scenarios/abort-during-catching-up.json | 2 + .../coordinator-death-and-recovery.json | 2 + .../scenarios/handoff-failure-kinds.json | 2 + .../scenarios/handoff-local-division.json | 77 ++++++++++++++++ conformance/scenarios/handoff-local-fold.json | 77 ++++++++++++++++ .../scenarios/handoff-local-step-aborted.json | 89 +++++++++++++++++++ conformance/scenarios/index.json | 50 ++++++++++- .../scenarios/migration-rate-control.json | 4 + .../scenarios/node-dies-mid-migration.json | 2 + .../plan-superseded-by-new-epoch.json | 6 ++ .../scenarios/quiesce-lease-expiry.json | 2 + .../scenarios/rebase-drops-a-handoff.json | 4 + .../scenarios/rebase-past-the-cutover.json | 4 + .../undetermined-resolves-both-ways.json | 2 + conformance/vectors/errors/taxonomy.json | 3 +- .../vectors/migration/plan-construction.json | 66 ++++++++++++-- docs/README.md | 2 +- docs/design/05-glossary.md | 3 + docs/design/10-specification.md | 74 +++++++++++++-- docs/design/30-conformance.md | 7 ++ ...0089-a-division-undone-by-a-combination.md | 71 +++++++++++++++ ...090-no-rebase-across-a-lineage-boundary.md | 49 ++++++++++ ports/java/conformance/report.txt | 12 +-- .../core/internal/migrate/Handoff.java | 39 ++++++++ .../core/internal/migrate/MigrationPlan.java | 81 +++++++++++++---- .../sharder/error/ErrorCode.java | 2 +- .../sharder/error/PlanRefusedException.java | 6 +- .../sharder/migrate/HandoffContext.java | 18 +++- .../sharder/migrate/HandoffState.java | 4 +- .../sharder/migrate/HookDeclaration.java | 17 +++- .../sharder/migrate/MovementHooks.java | 22 +++++ .../migrate/internal/DefaultCoordinator.java | 30 ++++++- .../internal/DefaultMigrationPlan.java | 27 +++++- .../sharder/api/MigrationTest.java | 2 +- .../sharder/conformance/CoreVectors.java | 14 ++- .../sharder/conformance/ScenarioRunner.java | 14 ++- 45 files changed, 1054 insertions(+), 106 deletions(-) create mode 100644 conformance/scenarios/handoff-local-division.json create mode 100644 conformance/scenarios/handoff-local-fold.json create mode 100644 conformance/scenarios/handoff-local-step-aborted.json create mode 100644 docs/design/adr/0089-a-division-undone-by-a-combination.md create mode 100644 docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md diff --git a/conformance/coverage.json b/conformance/coverage.json index 453cea9..72e385c 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -1,16 +1,16 @@ { "specification": "docs/design/10-specification.md", - "statedRequirements": 680, - "coveredRequirements": 468, - "uncoveredRequirements": 212, + "statedRequirements": 688, + "coveredRequirements": 472, + "uncoveredRequirements": 216, "coveragePercent": 68, "identifiersNamedButNotStated": [], "byPrefix": [ { "prefix": "LIN", "section": "", - "stated": 24, - "covered": 14, + "stated": 32, + "covered": 18, "uncovered": [ "LIN-001", "LIN-002", @@ -21,9 +21,13 @@ "LIN-031", "LIN-032", "LIN-034", - "LIN-043" + "LIN-043", + "LIN-053", + "LIN-054", + "LIN-055", + "LIN-058" ], - "percent": 58 + "percent": 56 }, { "prefix": "CFG", @@ -998,6 +1002,10 @@ "LIN-042", "LIN-044", "LIN-045", + "LIN-051", + "LIN-052", + "LIN-056", + "LIN-057", "MOVE-001", "MOVE-011", "MOVE-021", @@ -1069,8 +1077,8 @@ "TOPO-221", "TOPO-231" ], - "requirementsAtLevel": 94, - "requirementsWhenDeclared": 418 + "requirementsAtLevel": 98, + "requirementsWhenDeclared": 422 } ], "coveredBy": { @@ -1720,6 +1728,21 @@ "LIN-045": [ "vectors/migration/plan-construction.json" ], + "LIN-051": [ + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "vectors/migration/plan-construction.json" + ], + "LIN-052": [ + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "vectors/migration/plan-construction.json" + ], + "LIN-057": [ + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "vectors/migration/plan-construction.json" + ], "PROP-010": [ "properties/properties.json", "vectors/movement/add-one-node.json" @@ -2497,6 +2520,9 @@ "scenarios/abort-during-catching-up.json", "scenarios/handoff-failure-kinds.json", "scenarios/handoff-happy-path.json", + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "scenarios/handoff-local-step-aborted.json", "scenarios/node-dies-mid-migration.json", "scenarios/quiesce-lease-expiry.json", "scenarios/undetermined-resolves-both-ways.json" @@ -2505,6 +2531,8 @@ "properties/properties.json", "scenarios/handoff-failure-kinds.json", "scenarios/handoff-happy-path.json", + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", "scenarios/undetermined-resolves-both-ways.json" ], "MOVE-441": [ @@ -2650,7 +2678,9 @@ "scenarios/redirect-walk-depth-limit.json" ], "MOVE-001": [ - "scenarios/handoff-happy-path.json" + "scenarios/handoff-happy-path.json", + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json" ], "MOVE-041": [ "scenarios/handoff-happy-path.json" @@ -2685,6 +2715,15 @@ "scenarios/handoff-happy-path.json", "scenarios/quiesce-lease-expiry.json" ], + "LIN-056": [ + "scenarios/handoff-local-step-aborted.json" + ], + "MOVE-421": [ + "scenarios/abort-during-catching-up.json", + "scenarios/handoff-local-step-aborted.json", + "scenarios/node-dies-mid-migration.json", + "scenarios/rebase-drops-a-handoff.json" + ], "CFG-055": [ "scenarios/quiesce-lease-expiry.json" ], @@ -2714,11 +2753,6 @@ "HEALTH-048": [ "scenarios/node-dies-mid-migration.json" ], - "MOVE-421": [ - "scenarios/abort-during-catching-up.json", - "scenarios/node-dies-mid-migration.json", - "scenarios/rebase-drops-a-handoff.json" - ], "MOVE-481": [ "scenarios/node-dies-mid-migration.json", "scenarios/plan-superseded-by-new-epoch.json", diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index b3861a2..36e5c8d 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "3f56d0a573f1aaf7f364802e43a2bb0d7682eb08a8e7632eb89b86728c682070", + "revision": "0cbfa051871dc2e6467163cc79a276dc19769add35ba8e5c5fd14e5b6cab2312", "strategySurfaces": [ "directory", "rendezvous", @@ -42,8 +42,8 @@ "failures": 0 }, "scale": { - "wallTimeMs": 899, - "peakResidentBytes": 591892480, + "wallTimeMs": 892, + "peakResidentBytes": 602157056, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/generator/generate_extra.py b/conformance/generator/generate_extra.py index 3998bbe..fe39c61 100644 --- a/conformance/generator/generate_extra.py +++ b/conformance/generator/generate_extra.py @@ -83,7 +83,7 @@ def emit(root, path, vector_set, kind, description, requirements, cases, topolog (403, "handoffFailed", "no", "a handoff reached failed", "operator action, directed by the failure kind in cause; undetermined takes a" " re-observation", - ["unverified", "residue", "undetermined", "rollbackFailed"]), + ["unverified", "residue", "undetermined", "rollbackFailed", "undivided"]), ] @@ -520,13 +520,13 @@ def path(name): plans = [] for label, before, after, requirements, note in [ ("divided-source-from-the-parent", "delta-ring-before", "delta-ring-added", - ["LIN-041", "LIN-042", "LIN-045"], + ["LIN-041", "LIN-042", "LIN-045", "LIN-051", "LIN-052", "LIN-057"], "`LIN-041`: the contents of the divided shard `0000000000002000` are held by the " "replicas of its parent `0000000000003000`, so the handoff names one of them as its " "source. A plan built over the ownership delta alone has no entry for the parent, whose " "replica set did not change, and names a source equal to the destination."), ("folded-destination-outside-the-delta", "delta-ring-added", "delta-ring-removed", - ["LIN-041", "LIN-044", "LIN-045"], + ["LIN-041", "LIN-044", "LIN-045", "LIN-051", "LIN-052", "LIN-057"], "`LIN-044`: the shard that absorbs the folded extent keeps its replica set, so the " "ownership delta reports no entry for it, and a plan built over the delta alone moves " "nothing to the replica that does not hold the folded parent."), @@ -544,9 +544,10 @@ def path(name): emit(root, "vectors/migration/plan-construction.json", "migration-plan-construction", "planConstruction", "How a plan derives a handoff from a lineage: the source of a divided shard, the " - "destination of a fold that the ownership delta does not report, and the plan under the " - "identity lineage.", - ["LIN-041", "LIN-042", "LIN-044", "LIN-045"], plans, level="migration") + "destination of a fold that the ownership delta does not report, the local steps beside " + "them and their order, and the plan under the identity lineage.", + ["LIN-041", "LIN-042", "LIN-044", "LIN-045", "LIN-051", "LIN-052", "LIN-057"], plans, + level="migration") diff --git a/conformance/generator/generate_scenarios.py b/conformance/generator/generate_scenarios.py index 913cafa..8ad010d 100644 --- a/conformance/generator/generate_scenarios.py +++ b/conformance/generator/generate_scenarios.py @@ -57,6 +57,9 @@ "ejection-ceiling": "failover", "probation-ramp": "failover", "outlier-comparison-set": "failover", + "handoff-local-division": "migration", + "handoff-local-fold": "migration", + "handoff-local-step-aborted": "migration", "health-reset-on-reentry": "failover", } @@ -538,7 +541,8 @@ def plan_setup(plan): "toEpoch": plan.initial_target_epoch, "policy": dict(plan.policy), "handoffs": [ - {"id": handoff.id, "shard": handoff.shard, + {"id": handoff.id, "shard": handoff.shard, "sourceShard": handoff.source_shard, + "kind": handoff.kind, "source": handoff.source, "destination": handoff.destination} for handoff in plan.handoffs.values() ], @@ -613,6 +617,49 @@ def build_happy_path_scenario(): "MOVE-331", "MOVE-332", "MOVE-333"], steps) +def build_local_step_scenarios(): + """`LIN-051` through `LIN-058`: a division and a fold that happen on one node.""" + for name, kind, trigger, description in [ + ("handoff-local-division", "divide", "divideSuccess", + "A node that holds the parent under the earlier snapshot and the child under the later " + "one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to " + "`dividing` to `complete` rather than the sequence of `MOVE-021`."), + ("handoff-local-fold", "combine", "combineSuccess", + "The inverse: a node folds the copies it holds into one that matches the extent it now " + "owns, which is the step a merge ends with under `LIN-057`."), + ]: + plan = Plan(1, 2, "migration", [Handoff("l-1", "1", "n2", "n2", kind=kind, + source_shard="2")]) + steps = [] + for index, step_trigger in enumerate(["admittedByRatePolicyLocal", trigger]): + at = 1000 + index * 1000 + outcome = plan.step("l-1", step_trigger, at=at) + steps.append({"action": "handoffStep", "handoff": "l-1", "trigger": step_trigger, + "at": at, + "expect": {"outcome": outcome, "state": plan.state("l-1")}}) + steps.append({"action": "expectSummary", "expect": plan.summary()}) + register(name, description, + ["LIN-051", "LIN-052", "LIN-057", "MOVE-001", "MOVE-021", "MOVE-031"], steps, + setup=plan_setup(plan)) + + # `LIN-056`: an aborted local step is undone by the inverse hook, so `dividing` reaches + # `aborting` like every other state that does and `MOVE-233` keeps its unconditional form. + plan = Plan(1, 2, "migration", [Handoff("l-1", "1", "n2", "n2", kind="divide", + source_shard="2")]) + steps = [] + for index, step_trigger in enumerate(["admittedByRatePolicyLocal", "abort", + "rollbackSuccess"]): + at = 1000 + index * 1000 + outcome = plan.step("l-1", step_trigger, at=at) + steps.append({"action": "handoffStep", "handoff": "l-1", "trigger": step_trigger, + "at": at, "expect": {"outcome": outcome, "state": plan.state("l-1")}}) + steps.append({"action": "expectSummary", "expect": plan.summary()}) + register("handoff-local-step-aborted", + "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local " + "step is reversible and leaves the node holding the extent it held before.", + ["LIN-056", "MOVE-021", "MOVE-421"], steps, setup=plan_setup(plan)) + + def build_quiesce_lease_scenario(): """`MOVE-331` through `MOVE-336`: the commit horizon, and a lease too short to carry one.""" steps = [] @@ -1777,6 +1824,7 @@ def main(): build_split_view_scenario() build_redirect_scenario() build_happy_path_scenario() + build_local_step_scenarios() build_quiesce_lease_scenario() build_node_dies_scenario() build_abort_during_catchup_scenario() diff --git a/conformance/generator/property_definitions.py b/conformance/generator/property_definitions.py index d3ae635..1c3bfbf 100644 --- a/conformance/generator/property_definitions.py +++ b/conformance/generator/property_definitions.py @@ -447,14 +447,14 @@ "requirements": ["MOVE-011", "MOVE-021", "MOVE-031", "MOVE-441", "MOVE-491"], "level": "migration", "statement": "complete, aborted, and failed admit no transition out. A handoff in " - "failed carries exactly one of the four failure kinds. An abort is " + "failed carries exactly one of the failure kinds. An abort is " "idempotent and is not admitted at or beyond verifying.", "quantifier": "for every terminal state and every trigger", "sample": {"generator": "scenario", "count": "every trigger from every terminal state"}, "check": {"form": "invariant", "statement": "a transition out of a terminal state is refused, and a handoff " "in failed carries one kind from { unverified, residue, " - "undetermined, rollbackFailed }"}, + "undetermined, rollbackFailed, undivided }"}, "witness": "scenarios/handoff-failure-kinds.json", }, { diff --git a/conformance/generator/sharder_ref/handoff.py b/conformance/generator/sharder_ref/handoff.py index bc0a2f9..c7246dd 100644 --- a/conformance/generator/sharder_ref/handoff.py +++ b/conformance/generator/sharder_ref/handoff.py @@ -14,12 +14,21 @@ TERMINAL = ("complete", "aborted", "failed") -FAILURE_KINDS = ("unverified", "residue", "undetermined", "rollbackFailed") +FAILURE_KINDS = ("unverified", "residue", "undetermined", "rollbackFailed", + "undivided") # `MOVE-021`, exactly. (from, trigger) -> to TRANSITIONS = { ("planned", "admittedByRatePolicy"): "preparing", ("planned", "abort"): "aborted", + # `LIN-051`: a local step moves nothing between nodes, so it runs the short sequence. The + # admission trigger is the same one; which state it reaches is the handoff's kind. + ("planned", "admittedByRatePolicyLocal"): "dividing", + ("dividing", "divideSuccess"): "complete", + ("dividing", "combineSuccess"): "complete", + ("dividing", "abort"): "aborting", + ("dividing", "attemptsExhausted"): "aborting", + ("dividing", "dividePermanent"): "failed", ("preparing", "prepareSuccess"): "transferring", ("preparing", "abort"): "aborting", ("preparing", "attemptsExhausted"): "aborting", @@ -99,6 +108,8 @@ def retry_backoff(attempt): ("cutover", "commitUndetermined"): "undetermined", ("cleanup", "attemptsExhausted"): "residue", ("aborting", "attemptsExhausted"): "rollbackFailed", + # `MOVE-011`: a copy that matches neither the parent's extent nor the child's. + ("dividing", "dividePermanent"): "undivided", } @@ -107,9 +118,14 @@ class HandoffError(Exception): class Handoff: - def __init__(self, handoff_id, shard, source, destination, target_epoch=None): + def __init__(self, handoff_id, shard, source, destination, target_epoch=None, + kind="handoff", source_shard=None): self.id = handoff_id self.shard = shard + self.kind = kind + # `LIN-041`: the shard the contents come from, which differs from `shard` only where a + # lineage divided or folded an extent. + self.source_shard = source_shard or shard self.source = source self.destination = destination self.state = "planned" diff --git a/conformance/generator/sharder_ref/lineage.py b/conformance/generator/sharder_ref/lineage.py index 70571e9..bc9c621 100644 --- a/conformance/generator/sharder_ref/lineage.py +++ b/conformance/generator/sharder_ref/lineage.py @@ -191,12 +191,35 @@ def parents_of(before, after): def plan_handoffs(before, after, replicas_of): - """`LIN-041` through `LIN-045`: one handoff per parent a destination does not already hold.""" + """`LIN-041` through `LIN-058`: the handoffs of a plan and the local steps beside them. + + `LIN-057` fixes the order within one shard: a division is sequenced before every handoff that + draws from the divided parent, and a fold after every handoff that draws into the folded shard. + """ parents = parents_of(before, after) + kind = before.strategy["kind"] + identity = kind != "ring" or placement.shards(before) == placement.shards(after) + earlier = {} if identity else extents(before) + later = {} if identity else extents(after) + out = [] for shard in placement.shards(after): destinations = replicas_of(after, shard) - for parent in parents[shard]: + mine = parents[shard] + divisions, folds, moves = [], [], [] + if not identity and len(mine) == 1 and not equal(earlier[mine[0]], later[shard]): + # `LIN-052`: the extent narrowed, so a node holding both divides its own copy. + for node in destinations: + if node in replicas_of(before, mine[0]): + divisions.append({"shard": shard, "sourceShard": mine[0], "kind": "divide", + "source": node, "destination": node}) + if not identity and len(mine) > 1: + # `LIN-052`: the extent widened, so a node holding a parent folds what it holds. + for node in destinations: + if any(node in replicas_of(before, parent) for parent in mine): + folds.append({"shard": shard, "sourceShard": shard, "kind": "combine", + "source": node, "destination": node}) + for parent in mine: held = replicas_of(before, parent) if not held: continue @@ -205,10 +228,13 @@ def plan_handoffs(before, after, replicas_of): for position, destination in enumerate(needing): # `LIN-045`: a replica giving the shard up is drained before one keeping it. source = departing[position] if position < len(departing) else held[0] - out.append({"shard": shard, "sourceShard": parent, - "source": source, "destination": destination}) - for position, handoff in enumerate(out): - same = [h for h in out[:position] if h["shard"] == handoff["shard"]] - handoff["id"] = "h-%s" % handoff["shard"] if not same \ - else "h-%s-%d" % (handoff["shard"], len(same)) + moves.append({"shard": shard, "sourceShard": parent, "kind": "handoff", + "source": source, "destination": destination}) + out.extend(divisions + moves + folds) + + for position, entry in enumerate(out): + same = [e for e in out[:position] if e["shard"] == entry["shard"]] + prefix = "l-" if entry["kind"] != "handoff" else "h-" + entry["id"] = "%s%s" % (prefix, entry["shard"]) if not same \ + else "%s%s-%d" % (prefix, entry["shard"], len(same)) return out diff --git a/conformance/manifest.json b/conformance/manifest.json index a89347e..f4ddc25 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,9 +1,9 @@ { "suite": "sharder conformance suite", "revision": { - "id": "3f56d0a573f1aaf7f364802e43a2bb0d7682eb08a8e7632eb89b86728c682070", + "id": "0cbfa051871dc2e6467163cc79a276dc19769add35ba8e5c5fd14e5b6cab2312", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", - "fileCount": 201, + "fileCount": 204, "trees": [ "vectors", "topologies", @@ -18,10 +18,10 @@ "vectorFiles": 69, "vectorCases": 608, "topologyDocuments": 108, - "scenarios": 22, - "scenarioSteps": 303, + "scenarios": 25, + "scenarioSteps": 313, "properties": 28, - "requirementsNamed": 468 + "requirementsNamed": 472 }, "requirementsNamed": [ "CFG-014", @@ -203,6 +203,10 @@ "LIN-042", "LIN-044", "LIN-045", + "LIN-051", + "LIN-052", + "LIN-056", + "LIN-057", "MOVE-001", "MOVE-011", "MOVE-021", @@ -584,9 +588,9 @@ "surface": "the handoff coordinator and rate control", "vectorFiles": 5, "vectorCases": 25, - "scenarios": 12, + "scenarios": 15, "properties": 3, - "requirementsNamed": 94 + "requirementsNamed": 98 } ], "strategySurfaces": [ @@ -1744,7 +1748,7 @@ "ERR-021", "REPL-025" ], - "sha256": "16468a36a940a2a1fe2028e57b2da665f2800a403f2cee3552832d8822ae2373" + "sha256": "8e923beb7e6e5d839a6df17d214722e7ce37f960b70fb0f21e0e9591b9a5dcec" }, { "file": "vectors/formulas/attempt-limit.json", @@ -2080,16 +2084,19 @@ "ring", "slot" ], - "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, and the plan under the identity lineage.", + "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, the local steps beside them and their order, and the plan under the identity lineage.", "topology": null, "caseCount": 3, "requirements": [ "LIN-041", "LIN-042", "LIN-044", - "LIN-045" + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" ], - "sha256": "c19f77e44d8c9a6c686e0c6b508c666c3718fb2266f7755c63cee21d55aae2de" + "sha256": "97e1e1999b409306aa3ec272c8aad95aaeef61164012416a2e82f8eec034afbf" }, { "file": "vectors/movement/add-one-node.json", @@ -3257,6 +3264,48 @@ "TOPO-221" ] }, + { + "file": "scenarios/handoff-local-division.json", + "scenario": "handoff-local-division", + "level": "migration", + "description": "A node that holds the parent under the earlier snapshot and the child under the later one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to `dividing` to `complete` rather than the sequence of `MOVE-021`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-fold.json", + "scenario": "handoff-local-fold", + "level": "migration", + "description": "The inverse: a node folds the copies it holds into one that matches the extent it now owns, which is the step a merge ends with under `LIN-057`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-step-aborted.json", + "scenario": "handoff-local-step-aborted", + "level": "migration", + "description": "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local step is reversible and leaves the node holding the extent it held before.", + "stepCount": 4, + "requirements": [ + "LIN-056", + "MOVE-021", + "MOVE-421" + ] + }, { "file": "scenarios/quiesce-lease-expiry.json", "scenario": "quiesce-lease-expiry", diff --git a/conformance/properties/properties.json b/conformance/properties/properties.json index a5fbc3c..9a7d4c1 100644 --- a/conformance/properties/properties.json +++ b/conformance/properties/properties.json @@ -794,7 +794,7 @@ "MOVE-491" ], "level": "migration", - "statement": "complete, aborted, and failed admit no transition out. A handoff in failed carries exactly one of the four failure kinds. An abort is idempotent and is not admitted at or beyond verifying.", + "statement": "complete, aborted, and failed admit no transition out. A handoff in failed carries exactly one of the failure kinds. An abort is idempotent and is not admitted at or beyond verifying.", "quantifier": "for every terminal state and every trigger", "sample": { "generator": "scenario", @@ -802,7 +802,7 @@ }, "check": { "form": "invariant", - "statement": "a transition out of a terminal state is refused, and a handoff in failed carries one kind from { unverified, residue, undetermined, rollbackFailed }" + "statement": "a transition out of a terminal state is refused, and a handoff in failed carries one kind from { unverified, residue, undetermined, rollbackFailed, undivided }" }, "witness": "scenarios/handoff-failure-kinds.json" }, diff --git a/conformance/scenarios/abort-during-catching-up.json b/conformance/scenarios/abort-during-catching-up.json index fd6af0e..806ce35 100644 --- a/conformance/scenarios/abort-during-catching-up.json +++ b/conformance/scenarios/abort-during-catching-up.json @@ -25,6 +25,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/coordinator-death-and-recovery.json b/conformance/scenarios/coordinator-death-and-recovery.json index 0b0e14d..355905b 100644 --- a/conformance/scenarios/coordinator-death-and-recovery.json +++ b/conformance/scenarios/coordinator-death-and-recovery.json @@ -32,6 +32,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/handoff-failure-kinds.json b/conformance/scenarios/handoff-failure-kinds.json index c0aa3c0..e2981a5 100644 --- a/conformance/scenarios/handoff-failure-kinds.json +++ b/conformance/scenarios/handoff-failure-kinds.json @@ -30,6 +30,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/handoff-local-division.json b/conformance/scenarios/handoff-local-division.json new file mode 100644 index 0000000..efc1aea --- /dev/null +++ b/conformance/scenarios/handoff-local-division.json @@ -0,0 +1,77 @@ +{ + "scenario": "handoff-local-division", + "level": "migration", + "description": "A node that holds the parent under the earlier snapshot and the child under the later one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to `dividing` to `complete` rather than the sequence of `MOVE-021`.", + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ], + "deterministic": true, + "setup": { + "plan": { + "topologyId": "migration", + "fromEpoch": 1, + "toEpoch": 2, + "policy": { + "maxConcurrentHandoffs": 4, + "maxConcurrentPerSourceNode": 1, + "maxConcurrentPerDestinationNode": 1, + "initialStepBudget": 1, + "commitDeadlineMillis": 30000, + "quiesceLeaseMarginMillis": 1000 + }, + "handoffs": [ + { + "id": "l-1", + "shard": "1", + "sourceShard": "2", + "kind": "divide", + "source": "n2", + "destination": "n2" + } + ] + } + }, + "steps": [ + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "admittedByRatePolicyLocal", + "at": 1000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "planned", + "toState": "dividing" + }, + "state": "dividing" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "divideSuccess", + "at": 2000, + "expect": { + "outcome": { + "outcome": "settled", + "id": "l-1", + "terminalState": "complete", + "failureKind": null + }, + "state": "complete" + } + }, + { + "action": "expectSummary", + "expect": { + "complete": 1 + } + } + ] +} diff --git a/conformance/scenarios/handoff-local-fold.json b/conformance/scenarios/handoff-local-fold.json new file mode 100644 index 0000000..416ec65 --- /dev/null +++ b/conformance/scenarios/handoff-local-fold.json @@ -0,0 +1,77 @@ +{ + "scenario": "handoff-local-fold", + "level": "migration", + "description": "The inverse: a node folds the copies it holds into one that matches the extent it now owns, which is the step a merge ends with under `LIN-057`.", + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ], + "deterministic": true, + "setup": { + "plan": { + "topologyId": "migration", + "fromEpoch": 1, + "toEpoch": 2, + "policy": { + "maxConcurrentHandoffs": 4, + "maxConcurrentPerSourceNode": 1, + "maxConcurrentPerDestinationNode": 1, + "initialStepBudget": 1, + "commitDeadlineMillis": 30000, + "quiesceLeaseMarginMillis": 1000 + }, + "handoffs": [ + { + "id": "l-1", + "shard": "1", + "sourceShard": "2", + "kind": "combine", + "source": "n2", + "destination": "n2" + } + ] + } + }, + "steps": [ + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "admittedByRatePolicyLocal", + "at": 1000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "planned", + "toState": "dividing" + }, + "state": "dividing" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "combineSuccess", + "at": 2000, + "expect": { + "outcome": { + "outcome": "settled", + "id": "l-1", + "terminalState": "complete", + "failureKind": null + }, + "state": "complete" + } + }, + { + "action": "expectSummary", + "expect": { + "complete": 1 + } + } + ] +} diff --git a/conformance/scenarios/handoff-local-step-aborted.json b/conformance/scenarios/handoff-local-step-aborted.json new file mode 100644 index 0000000..8f71ad6 --- /dev/null +++ b/conformance/scenarios/handoff-local-step-aborted.json @@ -0,0 +1,89 @@ +{ + "scenario": "handoff-local-step-aborted", + "level": "migration", + "description": "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local step is reversible and leaves the node holding the extent it held before.", + "requirements": [ + "LIN-056", + "MOVE-021", + "MOVE-421" + ], + "deterministic": true, + "setup": { + "plan": { + "topologyId": "migration", + "fromEpoch": 1, + "toEpoch": 2, + "policy": { + "maxConcurrentHandoffs": 4, + "maxConcurrentPerSourceNode": 1, + "maxConcurrentPerDestinationNode": 1, + "initialStepBudget": 1, + "commitDeadlineMillis": 30000, + "quiesceLeaseMarginMillis": 1000 + }, + "handoffs": [ + { + "id": "l-1", + "shard": "1", + "sourceShard": "2", + "kind": "divide", + "source": "n2", + "destination": "n2" + } + ] + } + }, + "steps": [ + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "admittedByRatePolicyLocal", + "at": 1000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "planned", + "toState": "dividing" + }, + "state": "dividing" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "abort", + "at": 2000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "dividing", + "toState": "aborting" + }, + "state": "aborting" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "rollbackSuccess", + "at": 3000, + "expect": { + "outcome": { + "outcome": "settled", + "id": "l-1", + "terminalState": "aborted", + "failureKind": null + }, + "state": "aborted" + } + }, + { + "action": "expectSummary", + "expect": { + "aborted": 1 + } + } + ] +} diff --git a/conformance/scenarios/index.json b/conformance/scenarios/index.json index f92dfcd..f5f8603 100644 --- a/conformance/scenarios/index.json +++ b/conformance/scenarios/index.json @@ -1,7 +1,7 @@ { "suite": "sharder simulation scenarios", - "scenarioCount": 22, - "stepCount": 303, + "scenarioCount": 25, + "stepCount": 313, "requirementsCovered": [ "CFG-030", "CFG-040", @@ -77,6 +77,10 @@ "HEALTH-052", "HEALTH-053", "HEALTH-055", + "LIN-051", + "LIN-052", + "LIN-056", + "LIN-057", "MOVE-001", "MOVE-011", "MOVE-021", @@ -259,6 +263,48 @@ "TOPO-221" ] }, + { + "file": "scenarios/handoff-local-division.json", + "scenario": "handoff-local-division", + "level": "migration", + "description": "A node that holds the parent under the earlier snapshot and the child under the later one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to `dividing` to `complete` rather than the sequence of `MOVE-021`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-fold.json", + "scenario": "handoff-local-fold", + "level": "migration", + "description": "The inverse: a node folds the copies it holds into one that matches the extent it now owns, which is the step a merge ends with under `LIN-057`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-step-aborted.json", + "scenario": "handoff-local-step-aborted", + "level": "migration", + "description": "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local step is reversible and leaves the node holding the extent it held before.", + "stepCount": 4, + "requirements": [ + "LIN-056", + "MOVE-021", + "MOVE-421" + ] + }, { "file": "scenarios/quiesce-lease-expiry.json", "scenario": "quiesce-lease-expiry", diff --git a/conformance/scenarios/migration-rate-control.json b/conformance/scenarios/migration-rate-control.json index 5db3f0c..5f7e5d4 100644 --- a/conformance/scenarios/migration-rate-control.json +++ b/conformance/scenarios/migration-rate-control.json @@ -29,12 +29,16 @@ { "id": "h-a", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n4" }, { "id": "h-b", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n1", "destination": "n3" } diff --git a/conformance/scenarios/node-dies-mid-migration.json b/conformance/scenarios/node-dies-mid-migration.json index 6568fd5..1df4ad2 100644 --- a/conformance/scenarios/node-dies-mid-migration.json +++ b/conformance/scenarios/node-dies-mid-migration.json @@ -31,6 +31,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/plan-superseded-by-new-epoch.json b/conformance/scenarios/plan-superseded-by-new-epoch.json index 766c8a2..1a0347c 100644 --- a/conformance/scenarios/plan-superseded-by-new-epoch.json +++ b/conformance/scenarios/plan-superseded-by-new-epoch.json @@ -30,18 +30,24 @@ { "id": "h-a", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" }, { "id": "h-b", "shard": "2", + "sourceShard": "2", + "kind": "handoff", "source": "n3", "destination": "n1" }, { "id": "h-c", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n3" } diff --git a/conformance/scenarios/quiesce-lease-expiry.json b/conformance/scenarios/quiesce-lease-expiry.json index 0f89c87..0177867 100644 --- a/conformance/scenarios/quiesce-lease-expiry.json +++ b/conformance/scenarios/quiesce-lease-expiry.json @@ -30,6 +30,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/rebase-drops-a-handoff.json b/conformance/scenarios/rebase-drops-a-handoff.json index 383abb8..8733afe 100644 --- a/conformance/scenarios/rebase-drops-a-handoff.json +++ b/conformance/scenarios/rebase-drops-a-handoff.json @@ -33,12 +33,16 @@ { "id": "h-0", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n5" }, { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n3", "destination": "n6" } diff --git a/conformance/scenarios/rebase-past-the-cutover.json b/conformance/scenarios/rebase-past-the-cutover.json index e7e52ba..f02c84b 100644 --- a/conformance/scenarios/rebase-past-the-cutover.json +++ b/conformance/scenarios/rebase-past-the-cutover.json @@ -29,12 +29,16 @@ { "id": "h-0", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n5" }, { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n3", "destination": "n6" } diff --git a/conformance/scenarios/undetermined-resolves-both-ways.json b/conformance/scenarios/undetermined-resolves-both-ways.json index c684cf2..518e27d 100644 --- a/conformance/scenarios/undetermined-resolves-both-ways.json +++ b/conformance/scenarios/undetermined-resolves-both-ways.json @@ -37,6 +37,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/vectors/errors/taxonomy.json b/conformance/vectors/errors/taxonomy.json index 24184aa..a65fd4a 100644 --- a/conformance/vectors/errors/taxonomy.json +++ b/conformance/vectors/errors/taxonomy.json @@ -192,7 +192,8 @@ "unverified", "residue", "undetermined", - "rollbackFailed" + "rollbackFailed", + "undivided" ] } ], diff --git a/conformance/vectors/migration/plan-construction.json b/conformance/vectors/migration/plan-construction.json index 6bf81d5..1a56872 100644 --- a/conformance/vectors/migration/plan-construction.json +++ b/conformance/vectors/migration/plan-construction.json @@ -2,12 +2,15 @@ "vectorSet": "migration-plan-construction", "kind": "planConstruction", "level": "migration", - "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, and the plan under the identity lineage.", + "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, the local steps beside them and their order, and the plan under the identity lineage.", "requirements": [ "LIN-041", "LIN-042", "LIN-044", - "LIN-045" + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" ], "cases": [ { @@ -15,7 +18,10 @@ "requirements": [ "LIN-041", "LIN-042", - "LIN-045" + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" ], "before": "topologies/delta-ring-before.topology.json", "after": "topologies/delta-ring-added.topology.json", @@ -25,6 +31,7 @@ { "shard": "0000000000001000", "sourceShard": "0000000000001000", + "kind": "handoff", "source": "b", "destination": "c", "id": "h-0000000000001000" @@ -32,9 +39,34 @@ { "shard": "0000000000002000", "sourceShard": "0000000000003000", + "kind": "divide", + "source": "b", + "destination": "b", + "id": "l-0000000000002000" + }, + { + "shard": "0000000000002000", + "sourceShard": "0000000000003000", + "kind": "handoff", "source": "a", "destination": "c", - "id": "h-0000000000002000" + "id": "h-0000000000002000-1" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "divide", + "source": "b", + "destination": "b", + "id": "l-0000000000003000" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "divide", + "source": "a", + "destination": "a", + "id": "l-0000000000003000-1" } ] } @@ -44,7 +76,10 @@ "requirements": [ "LIN-041", "LIN-044", - "LIN-045" + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" ], "before": "topologies/delta-ring-added.topology.json", "after": "topologies/delta-ring-removed.topology.json", @@ -54,6 +89,7 @@ { "shard": "0000000000001000", "sourceShard": "0000000000001000", + "kind": "handoff", "source": "c", "destination": "b", "id": "h-0000000000001000" @@ -61,9 +97,26 @@ { "shard": "0000000000003000", "sourceShard": "0000000000002000", + "kind": "handoff", "source": "c", "destination": "a", "id": "h-0000000000003000" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "combine", + "source": "b", + "destination": "b", + "id": "l-0000000000003000-1" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "combine", + "source": "a", + "destination": "a", + "id": "l-0000000000003000-2" } ] } @@ -82,6 +135,7 @@ { "shard": "3", "sourceShard": "3", + "kind": "handoff", "source": "b", "destination": "d", "id": "h-3" @@ -89,6 +143,7 @@ { "shard": "4", "sourceShard": "4", + "kind": "handoff", "source": "b", "destination": "d", "id": "h-4" @@ -96,6 +151,7 @@ { "shard": "5", "sourceShard": "5", + "kind": "handoff", "source": "b", "destination": "d", "id": "h-5" diff --git a/docs/README.md b/docs/README.md index 8568647..bd19e87 100644 --- a/docs/README.md +++ b/docs/README.md @@ -197,7 +197,7 @@ other port carries too. | placement and weights | [0002](design/adr/0002-placement-strategy-set.md), [0003](design/adr/0003-integer-node-weights.md), [0010](design/adr/0010-shard-identifier-naming.md), [0011](design/adr/0011-derived-assignment-virtual-nodes.md), [0013](design/adr/0013-range-bounds-over-routing-key.md), [0039](design/adr/0039-placement-cost-model-and-warning-thresholds.md), [0043](design/adr/0043-assignment-mode-defaults.md), [0054](design/adr/0054-range-strategy-withdrawal.md), [0055](design/adr/0055-slot-derived-assignment-withdrawal.md), [0071](design/adr/0071-candidate-ordering-over-a-placement-ring.md), [0072](design/adr/0072-preparation-cost-reported-before-preparation.md) | | topology model and format | [0004](design/adr/0004-topology-provider-contract.md), [0005](design/adr/0005-epoch-and-version-semantics.md), [0006](design/adr/0006-failure-domain-model.md), [0008](design/adr/0008-json-canonical-serialisation.md), [0009](design/adr/0009-override-composition.md), [0038](design/adr/0038-provider-contract-in-the-specification.md), [0048](design/adr/0048-conditional-fetch-in-the-provider-contract.md) | | replication and failover | [0007](design/adr/0007-administrative-state-and-health-state.md), [0012](design/adr/0012-balance-bound-tolerances.md), [0014](design/adr/0014-read-affinity-as-a-separate-call.md), [0015](design/adr/0015-spread-degradation-algorithm.md), [0016](design/adr/0016-node-health-state-machine.md), [0017](design/adr/0017-failover-depth-and-substitution.md), [0036](design/adr/0036-spread-relaxation-ladder-direction.md), [0042](design/adr/0042-domain-path-scope.md), [0045](design/adr/0045-attempt-limit-resolution-order.md), [0046](design/adr/0046-bounded-routing-decision-surface.md), [0063](design/adr/0063-decision-api-surface-boundaries.md), [0069](design/adr/0069-per-level-occupancy-cap.md), [0070](design/adr/0070-spread-stage-feasibility-from-a-domain-count.md), [0075](design/adr/0075-affinity-path-length-refusal.md) | -| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md), [0086](design/adr/0086-shard-lineage-derived-from-extent.md), [0087](design/adr/0087-lineage-capability-derived-rather-than-declared.md), [0091](design/adr/0091-lineage-classification-as-a-signal.md) | +| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md), [0086](design/adr/0086-shard-lineage-derived-from-extent.md), [0087](design/adr/0087-lineage-capability-derived-rather-than-declared.md), [0089](design/adr/0089-a-division-undone-by-a-combination.md), [0090](design/adr/0090-no-rebase-across-a-lineage-boundary.md), [0091](design/adr/0091-lineage-classification-as-a-signal.md) | | cross-cutting contracts | [0024](design/adr/0024-closed-numbered-error-taxonomy.md), [0025](design/adr/0025-observability-contract-and-explain-record.md), [0026](design/adr/0026-configuration-defaults-and-locality.md), [0027](design/adr/0027-hash-seed-exposure-and-tenancy.md), [0060](design/adr/0060-node-label-cardinality-ceiling.md), [0076](design/adr/0076-ordered-rows-in-a-precedence-table.md) | | Java binding | [0028](design/adr/0028-java-module-and-artifact-layout.md), [0029](design/adr/0029-exception-idiom-for-the-taxonomy.md), [0031](design/adr/0031-jdk-baseline.md), [0032](design/adr/0032-dependency-free-json-and-canonicalisation.md), [0033](design/adr/0033-opaque-identifier-value-types.md), [0034](design/adr/0034-lazy-candidate-traversal-surface.md), [0041](design/adr/0041-exact-product-comparison-surface.md), [0081](design/adr/0081-single-java-module.md), [0085](design/adr/0085-hook-declarations-and-refused-aborts.md) | | conformance | [0035](design/adr/0035-manifest-driven-conformance-harness.md), [0037](design/adr/0037-specification-defect-repairs.md), [0052](design/adr/0052-conformance-level-partition.md), [0058](design/adr/0058-conformance-surfaces.md), [0059](design/adr/0059-place-conformance-level.md), [0061](design/adr/0061-suite-revision-identifier.md), [0064](design/adr/0064-hash-verification-before-generation.md), [0065](design/adr/0065-level-coverage-inside-surface-boundaries.md), [0077](design/adr/0077-scale-conformance-level.md), [0078](design/adr/0078-observability-contract-as-data.md), [0084](design/adr/0084-missing-member-as-a-validation-error.md), [0088](design/adr/0088-lineage-inside-the-migration-surface.md) | diff --git a/docs/design/05-glossary.md b/docs/design/05-glossary.md index 95a6eb5..d169226 100644 --- a/docs/design/05-glossary.md +++ b/docs/design/05-glossary.md @@ -38,6 +38,9 @@ the design states, and the Java port renders each one. next epoch. The authority publishes the change; the library classifies it and plans it. - **merge**. A change in which the extents of two or more shards become the extent of one shard of the next epoch. A merge is the reverse of a split. +- **local step**. The part of a split or a merge that happens on one node, where that node holds + the parent under the earlier snapshot and the child under the later one. Nothing moves between + nodes; the node divides or folds its own copy so that what it holds matches the extent it owns. - **partition**. A synonym for shard in external literature. The corpus uses shard. - **slot**. A shard produced by dividing the keyspace into a fixed count of numbered parts by modular arithmetic over the key hash. diff --git a/docs/design/10-specification.md b/docs/design/10-specification.md index 8b130e0..8cd2f23 100644 --- a/docs/design/10-specification.md +++ b/docs/design/10-specification.md @@ -3079,6 +3079,54 @@ up is drained in preference to one that is keeping it, and a plan is a pure func snapshots and the policy under `MOVE-221`, so the choice MUST be stated rather than left to an implementation. +#### The local division step + +`LIN-051`. Where a shard `s` of the later snapshot has a parent `p` whose extent differs from +`s`'s, and a node holds `p` under the earlier snapshot and `s` under the later one, a plan MUST +carry one local step for that node. The node already holds the contents, so nothing moves +between nodes, and what it has to do is divide or fold its own copy so that what it holds matches +the extent of `s`. A local step names the same node as its source and its destination, which is the one +case `LIN-042` does not govern, because no handoff of contents is described. + +`LIN-052`. A local step MUST call `divide` where the extent of `s` is strictly contained in the +extent of `p`, and `combine` where the extent of `s` contains the extents of two or more parents. +An implementation MUST call one local hook per local step and MUST NOT call both. + +`LIN-053`. An implementation MUST refuse a plan that requires a local step where `declare` answers +`supportsLineage` of false, reporting `planRefused` under `ERR-050` with the cause +`lineageUnsupported`. Whether an integrator's storage can divide a copy in place is a property of +that storage, which the library cannot observe, so `MOVE-111` declares it beside `supportsRollback` +and `supportsVerify` and the plan is refused rather than calling a hook that was never implemented. +A refusal at planning is the only useful moment: a plan that reached `dividing` and found no hook +would have no way forward and no way back. + +`LIN-054`. A local step MUST be retried under the same attempt rules as any other hook, and an +implementation MUST pass the attempt number in the context so a hook that already divided its copy +can answer success rather than dividing twice. An implementation MUST NOT require a local hook to +be idempotent beyond answering success for work it has already done. + +`LIN-055`. An implementation MUST NOT sequence a local step onto a node that does not hold every +shard the step names under the snapshot the step names it. A local step for `s` drawn from `p` is +sequenced only where the node's replica set membership holds under both snapshots, which is the +condition `LIN-051` states, and a plan MUST NOT admit one otherwise. + +`LIN-056`. A local step in `dividing` that is aborted MUST be undone by the inverse hook: a +division is undone by `combine` and a fold by `divide`, called through `rollback` under +`MOVE-421`. A local step is therefore reversible like every other state that reaches `aborting`, +and `MOVE-233` keeps the unconditional form it has. Where the inverse fails and its attempts are +exhausted, the handoff reaches `failed` with the kind `undivided` under `MOVE-011`. + +`LIN-057`. An implementation MUST order a local step against the handoffs of the same shard: a +division MUST be sequenced before any handoff that draws from the divided parent, and a fold MUST be +sequenced after every handoff that draws into the folded shard. A destination copying from a parent +that is being divided underneath it would copy an extent that is changing, and a node folding a copy +before the other parents have arrived would fold an incomplete one. + +`LIN-058`. A local step MUST count against the concurrency the migration policy of `RATE-011` +admits, in the same way a handoff does. A local step performs storage work on a node, which is the +thing the policy exists to bound, and an implementation MUST NOT admit one outside +`maxConcurrentHandoffs`. + ### Shard ownership handoff A handoff moves ownership of one shard from a source node to a destination node. The library @@ -3097,6 +3145,7 @@ storage protocol. | `catchingUp` | the residue accumulated during the copy is being closed | | `cutover` | the source is quiescing and the cutover record is being committed | | `verifying` | the destination copy is being checked against the source | +| `dividing` | a node that holds both the parent and the child is dividing or folding its own copy | | `cleanup` | the source copy is being released | | `complete` | terminal, ownership moved and the source released | | `aborting` | compensation is running after an abort | @@ -3104,7 +3153,10 @@ storage protocol. | `failed` | terminal, operator action is required | `MOVE-011`. A handoff in `failed` MUST carry exactly one failure kind from the set `unverified`, -`residue`, `undetermined`, and `rollbackFailed`. +`residue`, `undetermined`, `rollbackFailed`, and `undivided`. A kind of `undivided` states that a +local step under `LIN-051` neither completed nor was undone, so the node holds a copy that matches +neither the parent's extent nor the child's. None of the other four describes that state: each names +a condition of a copy that moved between nodes, and this one never left the node it is on. #### Transitions @@ -3115,6 +3167,10 @@ storage protocol. | none | `planned` | plan construction admits the shard | | `planned` | `preparing` | the rate policy admits the handoff | | `planned` | `aborted` | abort requested, the plan is superseded, or a rebase drops the handoff | +| `planned` | `dividing` | the rate policy admits a local step under `LIN-051` | +| `dividing` | `complete` | `divide` or `combine` returns success | +| `dividing` | `aborting` | abort requested, plan superseded or rebased past it, or attempts exhausted | +| `dividing` | `failed` | `divide` or `combine` reports a permanent failure | | `preparing` | `transferring` | `prepare` returns success | | `preparing` | `aborting` | abort requested, plan superseded or rebased past it, or attempts exhausted | | `transferring` | `catchingUp` | `transfer` reports no bulk remaining | @@ -3300,9 +3356,11 @@ interface MovementHooks: cleanup(ctx) -> HookResult rollback(ctx) -> HookResult observe(ctx) -> ObserveResult + divide(ctx) -> HookResult + combine(ctx) -> HookResult HandoffContext ctx = { - shardId, topologyId, fromEpoch, toEpoch, + shardId, sourceShardId, topologyId, fromEpoch, toEpoch, source: NodeId, destination: NodeId, attempt: u32, deadlineMillis: u32 } @@ -3310,7 +3368,8 @@ HandoffContext ctx = { HookDeclaration = { budgetUnit: string, # opaque to the library supportsRollback: boolean, - supportsVerify: boolean + supportsVerify: boolean, + supportsLineage: boolean } HookResult = one of { success, deferred(retryAfterMillis), retryable(reason), @@ -3579,9 +3638,12 @@ guarantees. `MOVE-411`. An abort requested in `planned` MUST move the handoff directly to `aborted` without calling a hook. No hook has run, so nothing is to compensate. -`MOVE-421`. An abort requested in `preparing`, `transferring`, or `catchingUp` MUST move the handoff -to `aborting` and MUST call `rollback`, whose duty is to release the destination's partial copy and -any scratch state `prepare` created. The source is untouched throughout, so the abort loses nothing. +`MOVE-421`. An abort requested in `preparing`, `transferring`, `catchingUp`, or `dividing` MUST move +the handoff to `aborting` and MUST call `rollback`, whose duty is to release the destination's +partial copy and any scratch state `prepare` created. From the first three the source is untouched +throughout, so the abort loses nothing. From `dividing` there is no second copy to release, and +`rollback` MUST restore the extent the node held before the step, which `LIN-056` states is the +inverse hook. `MOVE-431`. An abort requested in `cutover` MUST be admitted only while no cutover record belonging to the handoff exists under `MOVE-102`. The coordinator MUST call `observe` before diff --git a/docs/design/30-conformance.md b/docs/design/30-conformance.md index de575f0..ee232a0 100644 --- a/docs/design/30-conformance.md +++ b/docs/design/30-conformance.md @@ -770,6 +770,13 @@ what a lineage costs, which is the case `Placement cost` describes. `LIN-015` binds a registered strategy outside the core set. The suite carries core-set documents, so no data file can present one, and the same is true of every requirement about a registered strategy. +`LIN-053`, `LIN-054`, and `LIN-058` each read something outside the two snapshots: a hook +declaration, an attempt count, and the concurrency a policy admits. The first is the case +`Movement hooks` describes, and the other two are the case `Rate control and measurement` +describes. The state sequence a local step runs is an output, so +`scenarios/handoff-local-division.json`, `scenarios/handoff-local-fold.json`, and +`scenarios/handoff-local-step-aborted.json` carry `LIN-051`, `LIN-052`, `LIN-056`, and `LIN-057`. + `LIN-043` states that a plan emits no handoff for a shard with no parent. A fresh extent arises only where a later snapshot admits routing keys that the earlier one matched to no shard, which is the `directory` no-match of `DIR-010`, and `LIN-013` refuses a `directory` pair whose shard sets differ diff --git a/docs/design/adr/0089-a-division-undone-by-a-combination.md b/docs/design/adr/0089-a-division-undone-by-a-combination.md new file mode 100644 index 0000000..f50208c --- /dev/null +++ b/docs/design/adr/0089-a-division-undone-by-a-combination.md @@ -0,0 +1,71 @@ +# 0089. A division undone by a combination + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0086`](0086-shard-lineage-derived-from-extent.md) makes a division plannable, and a division that +happens entirely on one node is the part with no precedent in the state machine. A node that holds +the parent under the earlier snapshot and the child under the later one moves nothing to anybody. It +has to divide its own copy so that what it holds matches the extent it now owns. + +[`0022`](0022-range-split-lineage.md) had this case and answered it in a way +[`0054`](0054-range-strategy-withdrawal.md) was glad to be rid of. `SPLIT-171` required a handoff +whose local split had already succeeded to move to `failed(undetermined)` without a re-observation, +because the data no longer matched the parent's bounds and could not be put back. That was the +single carve-out in `MOVE-233`, and `0054` recorded its removal as a gain: +[`0051`](0051-recovery-from-an-undetermined-cutover.md) wanted `MOVE-233` unconditional, and with +`SPLIT` withdrawn it became so. + +Reinstating the local step reopens the question. If a succeeded division cannot be undone, the +carve-out comes back with it. + +## Decision + +A division is reversible, and the inverse of a division is a combination. + +`LIN-056` requires an aborted local step to be undone by the inverse hook: a division is undone by +`combine`, a fold by `divide`. A local step therefore reaches `aborting` like every other state that +does, `MOVE-421` gains `dividing` beside the three states it already names, and `MOVE-233` keeps the +unconditional form `0051` wanted. There is no carve-out. + +The cost lands on the integrator, visibly. `LIN-053` refuses a plan that requires a local step where +`declare` answers `supportsLineage` of false, which is the shape +[`0085`](0085-hook-declarations-and-refused-aborts.md) established: a declaration is the +integrator's statement about their own storage, and the library refuses rather than calling a hook +that was never implemented. + +A handoff whose inverse fails and whose attempts are exhausted reaches `failed` with the new kind +`undivided`. The four kinds `MOVE-011` already carries each name a condition of a copy that moved +between nodes, and this one never left the node it is on. + +## Consequences + +An integrator whose storage can divide a shard but not recombine two adjacent ones declares +`supportsLineage` of false and gets a refused plan rather than a handoff that can only fail. That is +a real restriction, and it is the honest one: a step that cannot be undone is not an abortable step, +and pretending otherwise is what produced `SPLIT-171`. + +The state machine gains one state and four transitions. `MOVE-021` says "exactly these transitions +and no others", so this is the largest single edit in the batch, and it lands after the lineage +computation is settled and vector-backed rather than beside it. + +A division whose children keep the parent's replica set is a local step alone, with no data crossing +the network, which is `0022`'s own observation and still true. + +## Alternatives + +A succeeded local step that cannot be undone, as `SPLIT-171` had it. Rejected. It reinstates the +carve-out in `MOVE-233` that `0051` argued against and `0054` removed, and it makes one state in the +machine behave unlike every other. + +Requiring no local hook at all, and asking the integrator to divide its copy out of band. Rejected. +The library would have no way to know whether the division had happened, so it could not sequence a +handoff that draws from the divided parent, which `LIN-057` has to order. + +Treating a local step as a handoff from the node to itself with the ordinary hook sequence. +Rejected. `prepare`, `transfer`, `catchUp`, `quiesce`, and `commitCutover` all describe two parties, +and a single-winner cutover between a node and itself is meaningless. The shorter path through +`dividing` says what actually happens. diff --git a/docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md b/docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md new file mode 100644 index 0000000..e35c1f3 --- /dev/null +++ b/docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md @@ -0,0 +1,49 @@ +# 0090. No rebase across a lineage boundary + +Date: 2026-09-19 + +Status: accepted + +## Context + +`MOVE-096` makes a handoff rebasable onto a newer snapshot exactly when that snapshot enumerates the +handoff's shard, its source, and its destination. +[`0050`](0050-plan-rebase-onto-a-newer-snapshot.md) settled that rule before a lineage existed, when +the only way a shard could leave a snapshot was for the authority to stop naming it. + +With [`0086`](0086-shard-lineage-derived-from-extent.md) a shard can leave a snapshot because its +extent was divided again. A handoff moving contents into a shard that a third epoch has since +divided has a shard the newer snapshot does not enumerate, so `MOVE-096` drops it. + +The question is whether the rebase should instead follow the lineage: recognise that the shard's +extent still exists under new names, and rewrite the handoff onto the children. + +## Decision + +It should not. A handoff whose shard the newer snapshot does not enumerate is aborted and +compensated, exactly as `MOVE-096` already has it. No `MOVE-09*` requirement changes. + +## Consequences + +A split overtaken mid-flight by a further split of the same region costs the work already done. The +destination's partial copy is released by `rollback` and the newer plan starts the move again, which +is correct but not free. + +The behaviour is what the port already does, so nothing in the coordinator changes and no scenario +moves. + +Relaxing a refusal later is cheap and withdrawing a clever rebase that proved wrong is not. A +transitive rebase would have to decide which child inherits a partially transferred copy, whether a +quiesce lease taken against the parent still binds the children, and what a cutover record naming +the parent means once the parent is gone. None of those has an obvious answer, and getting one wrong +loses data rather than time. + +## Alternatives + +Rewriting a handoff onto the children of its shard. Rejected above: it is a larger change than it +looks, it is not reversible once shipped, and the case it optimises is an authority publishing two +divisions of one region in quick succession, which is rare and already correct if slower. + +Refusing the rebase outright rather than aborting the handoff. Rejected. `MOVE-096` classifies each +handoff independently and a plan may contain many, so refusing the whole rebase because one handoff +cannot be carried would abandon the ones that can. diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index d04c51c..b170649 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,17 +1,17 @@ sharder conformance report, Java port -suite revision 3f56d0a573f1aaf7f364802e43a2bb0d7682eb08a8e7632eb89b86728c682070 +suite revision 0cbfa051871dc2e6467163cc79a276dc19769add35ba8e5c5fd14e5b6cab2312 hash 2 of 2 vector files, 34 of 34 cases, 0.01s place 45 of 45 vector files, 291 of 291 cases, 0.09s - core 8 of 8 vector files, 135 of 135 cases, 8.02s - scale 2 of 2 vector files, 2 of 2 cases, 0.90s + core 8 of 8 vector files, 135 of 135 cases, 8.07s + scale 2 of 2 vector files, 2 of 2 cases, 0.89s failover 3 of 3 vector files, 67 of 67 cases, 0.00s readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s fencing 3 of 3 vector files, 8 of 8 cases, 0.00s migration 5 of 5 vector files, 25 of 25 cases, 0.01s 608 cases, 0 failures - scale wall time 0.90s - scale wall time 899 ms - peak resident size 591892480 bytes (564 MiB) + scale wall time 0.89s + scale wall time 892 ms + peak resident size 602157056 bytes (574 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java index 9debdf3..e60364a 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java @@ -14,9 +14,37 @@ */ public final class Handoff { + /** + * Whether contents cross the network, or the node divides or folds a copy it already holds. + * + *

A local step under {@code LIN-051} names one node as both its source and its destination + * and runs the shorter sequence through {@code dividing}, because there is no second party to + * prepare, quiesce, or cut over to. + */ + public enum Kind { + HANDOFF("handoff"), DIVIDE("divide"), COMBINE("combine"); + + private final String spelling; + + Kind(String spelling) { + this.spelling = spelling; + } + + /** The spelling a vector joins on. */ + public String spelling() { + return spelling; + } + + /** Whether this is a local step rather than a move between nodes. */ + public boolean local() { + return this != HANDOFF; + } + } + private final String id; private final String shard; private final String sourceShard; + private final Kind kind; private final NodeId source; private final NodeId destination; private final long fromEpoch; @@ -33,9 +61,15 @@ public final class Handoff { Handoff(String id, String shard, String sourceShard, NodeId source, NodeId destination, long fromEpoch, long toEpoch) { + this(id, shard, sourceShard, Kind.HANDOFF, source, destination, fromEpoch, toEpoch); + } + + Handoff(String id, String shard, String sourceShard, Kind kind, NodeId source, + NodeId destination, long fromEpoch, long toEpoch) { this.id = id; this.shard = shard; this.sourceShard = sourceShard; + this.kind = kind; this.source = source; this.destination = destination; this.fromEpoch = fromEpoch; @@ -64,6 +98,11 @@ public String sourceShard() { return sourceShard; } + /** Whether contents move between nodes, or a node divides a copy it holds. */ + public Kind kind() { + return kind; + } + /** The node that owns the shard before the cutover. */ public NodeId source() { return source; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java index 0c24300..e0d0e7e 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java @@ -94,6 +94,13 @@ public static Handoff handoffOf(String id, String shard, NodeId source, NodeId d return new Handoff(id, shard, source, destination, fromEpoch, toEpoch); } + /** The same, carrying the parent and the kind a lineage gives it under {@code LIN-041}. */ + public static Handoff handoffOf(String id, String shard, String sourceShard, Handoff.Kind kind, + NodeId source, NodeId destination, long fromEpoch, + long toEpoch) { + return new Handoff(id, shard, sourceShard, kind, source, destination, fromEpoch, toEpoch); + } + /** * The plan between two snapshots, derived from the lineage under {@code LIN-041}. * @@ -116,9 +123,26 @@ public static MigrationPlan of(PlacementEngine from, PlacementEngine to, long quiesceLeaseMarginMillis) { MigrationPlan plan = new MigrationPlan(quiesceLeaseMarginMillis, to.document().epoch()); Map> parents = ShardExtents.parents(from, to); + Map classes = new LinkedHashMap<>(); + for (ShardExtents.Entry entry : ShardExtents.classify(from, to, OwnershipDelta::replicas)) { + classes.put(entry.shard(), entry.lineage()); + } for (String shard : to.placement().shards()) { List destinations = OwnershipDelta.replicas(to, shard); - for (String parent : parents.getOrDefault(shard, List.of(shard))) { + List drawn = parents.getOrDefault(shard, List.of(shard)); + ShardExtents.Lineage lineage = classes.get(shard); + + // LIN-057: a division is sequenced before every handoff that draws from the divided + // parent, and a fold after every handoff that draws into the folded shard. + if (lineage == ShardExtents.Lineage.DIVIDED) { + String parent = drawn.get(0); + for (NodeId node : destinations) { + if (OwnershipDelta.replicas(from, parent).contains(node)) { + admit(plan, shard, parent, Handoff.Kind.DIVIDE, node, node, from, to); + } + } + } + for (String parent : drawn) { List held = OwnershipDelta.replicas(from, parent); if (held.isEmpty()) { // LIN-043: a shard with no parent has no contents to move. @@ -130,26 +154,40 @@ public static MigrationPlan of(PlacementEngine from, PlacementEngine to, departing.removeAll(destinations); for (int entry = 0; entry < needing.size(); entry++) { // LIN-045: a replica giving the shard up is drained in preference to one - // keeping it, so a plan rebuilt from the same pair of snapshots names the same - // source for the same destination. - NodeId source = entry < departing.size() - ? departing.get(entry) - : held.get(0); - NodeId destination = needing.get(entry); - String id = "h-" + shard; - int suffix = 1; - while (plan.handoffs.containsKey(id)) { - id = "h-" + shard + "-" + suffix; - suffix++; + // keeping it, so a plan rebuilt from the same snapshots names the same source. + NodeId source = entry < departing.size() ? departing.get(entry) : held.get(0); + admit(plan, shard, parent, Handoff.Kind.HANDOFF, source, needing.get(entry), + from, to); + } + } + if (lineage == ShardExtents.Lineage.MERGED) { + for (NodeId node : destinations) { + boolean holdsAParent = drawn.stream() + .anyMatch(parent -> OwnershipDelta.replicas(from, parent).contains(node)); + if (holdsAParent) { + admit(plan, shard, shard, Handoff.Kind.COMBINE, node, node, from, to); } - plan.handoffs.put(id, new Handoff(id, shard, parent, source, destination, - from.document().epoch(), to.document().epoch())); } } } return plan; } + /** One entry of a plan, named so that a rebuilt plan names it identically under MOVE-221. */ + private static void admit(MigrationPlan plan, String shard, String sourceShard, + Handoff.Kind kind, NodeId source, NodeId destination, + PlacementEngine from, PlacementEngine to) { + String prefix = kind.local() ? "l-" : "h-"; + // The suffix counts the entries already admitted for this shard whatever their kind, so a + // local step and the handoff beside it never collide and the numbering is positional. + long already = plan.handoffs.values().stream() + .filter(existing -> existing.shard().equals(shard)) + .count(); + String id = already == 0 ? prefix + shard : prefix + shard + "-" + already; + plan.handoffs.put(id, new Handoff(id, shard, sourceShard, kind, source, destination, + from.document().epoch(), to.document().epoch())); + } + /** The handoffs of the plan, in the order it admitted them. */ public List handoffs() { return List.copyOf(handoffs.keySet()); @@ -273,7 +311,14 @@ public StepOutcome step(String id, String trigger, long at, String pressure) { } HandoffState to = switch (trigger) { case "admittedByRatePolicy" -> require(from, HandoffState.PLANNED, - HandoffState.PREPARING); + handoff(id).kind().local() ? HandoffState.DIVIDING : HandoffState.PREPARING); + // LIN-051: the same admission, named for the state it reaches, which a scenario + // drives by trigger rather than by reading the handoff's kind. + case "admittedByRatePolicyLocal" -> require(from, HandoffState.PLANNED, + HandoffState.DIVIDING); + // LIN-051: a local step moves nothing between nodes, so it runs the short sequence. + case "divideSuccess", "combineSuccess" -> require(from, HandoffState.DIVIDING, + HandoffState.COMPLETE); case "prepareSuccess" -> require(from, HandoffState.PREPARING, HandoffState.TRANSFERRING); case "noBulkRemaining" -> require(from, HandoffState.TRANSFERRING, @@ -293,7 +338,9 @@ public StepOutcome step(String id, String trigger, long at, String pressure) { ? HandoffState.ABORTED : HandoffState.ABORTING; // MOVE-021: attempts exhausted short of the cutover is an abort rather than a failure. case "attemptsExhausted" -> switch (from) { - case PREPARING, TRANSFERRING, CATCHING_UP -> HandoffState.ABORTING; + // MOVE-421: a local step is undone by the inverse hook, so it aborts like the + // three states that already do, under LIN-056. + case PREPARING, TRANSFERRING, CATCHING_UP, DIVIDING -> HandoffState.ABORTING; default -> null; }; case "quiesceExpired" -> require(from, HandoffState.CUTOVER, HandoffState.ABORTING); @@ -317,6 +364,8 @@ private StepOutcome failure(Handoff handoff, HandoffState from, String trigger) String kind = switch (trigger) { case "verifyMismatch" -> "unverified"; case "commitUndetermined" -> "undetermined"; + // MOVE-011: a copy that matches neither the parent's extent nor the child's. + case "dividePermanent" -> "undivided"; // MOVE-011: the kind names which copy the failure leaves at risk, so an exhausted // attempt budget means one thing in verifying, another in cleanup, and another while // compensation runs. diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java b/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java index 6cd1e6b..93f1656 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java @@ -170,7 +170,7 @@ public List causes() { "strategyUnsupported", "destinationOutsidePlacementSet", "policyInvalid", "topologyMismatch", "unalignedLineage", "lineageUnsupported"); case HANDOFF_FAILED -> List.of("unverified", "residue", "undetermined", - "rollbackFailed"); + "rollbackFailed", "undivided"); default -> List.of(); }; } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java b/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java index 5193d47..cd4d60f 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java @@ -23,7 +23,11 @@ public enum Cause { /** The policy carries a value outside its range. */ POLICY_INVALID("policyInvalid"), /** The plan names a topology the router does not hold. */ - TOPOLOGY_MISMATCH("topologyMismatch"); + TOPOLOGY_MISMATCH("topologyMismatch"), + /** The change moves a boundary without dividing or folding an extent, {@code LIN-022}. */ + UNALIGNED_LINEAGE("unalignedLineage"), + /** A local step is needed and the storage declares none, under {@code LIN-053}. */ + LINEAGE_UNSUPPORTED("lineageUnsupported"); private final String spelling; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java index 4c4141a..89d2e3c 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java @@ -10,6 +10,20 @@ * entered {@code cutover} keeps the epoch it held at that transition and runs to a terminal state * under it, under {@code MOVE-099}, so the two differ after a rebase. */ -public record HandoffContext(ShardId shardId, String topologyId, long fromEpoch, long toEpoch, - NodeId source, NodeId destination, int attempt, int deadlineMillis) { +public record HandoffContext(ShardId shardId, ShardId sourceShardId, String topologyId, + long fromEpoch, long toEpoch, NodeId source, NodeId destination, + int attempt, int deadlineMillis) { + + /** + * The context of a handoff whose contents come from the shard itself. + * + *

{@code sourceShardId} differs from {@code shardId} only where a lineage divided or folded + * an extent, under {@code LIN-041}. A hook that read {@code shardId} alone would search the + * source for a shard the source does not hold. + */ + public HandoffContext(ShardId shardId, String topologyId, long fromEpoch, long toEpoch, + NodeId source, NodeId destination, int attempt, int deadlineMillis) { + this(shardId, shardId, topologyId, fromEpoch, toEpoch, source, destination, attempt, + deadlineMillis); + } } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java index 014a5b2..22d3499 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java @@ -1,7 +1,7 @@ package com.codeheadsystems.sharder.migrate; /** - * The eleven handoff states of {@code MOVE-001}. + * The twelve handoff states of {@code MOVE-001}. * *

{@code complete}, {@code aborted}, and {@code failed} are terminal, and nothing transitions * out of one except the re-observation of {@code MOVE-233}, which the integrator calls for one @@ -10,6 +10,8 @@ public enum HandoffState { /** Admitted to the plan, no hook called. */ PLANNED("planned"), + /** A node holding both the parent and the child is dividing or folding its own copy. */ + DIVIDING("dividing"), /** The destination is being made ready to receive. */ PREPARING("preparing"), /** The bulk contents are being copied. */ diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java index 4ab3ea7..a89282a 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java @@ -6,12 +6,23 @@ *

{@code budgetUnit} is opaque to the library: it is reported and never interpreted, under * {@code MOVE-141}. {@code supportsVerify} of false is what {@code MOVE-181} reads when it decides * whether {@code cleanup} may follow a committed cutover directly. + * + *

{@code supportsLineage} states whether the integrator's storage can divide a copy in place and + * fold two adjacent copies back together. It is a property of that storage rather than of this + * library, which is why it is declared here rather than derived from the strategy: {@code LIN-053} + * refuses a plan that needs a local step where it is false, in preference to calling a hook that + * was never implemented. */ public record HookDeclaration(String budgetUnit, boolean supportsRollback, - boolean supportsVerify) { + boolean supportsVerify, boolean supportsLineage) { - /** A declaration in units of {@code budgetUnit} supporting both rollback and verification. */ + /** A declaration in units of {@code budgetUnit} supporting rollback and verification. */ public static HookDeclaration of(String budgetUnit) { - return new HookDeclaration(budgetUnit, true, true); + return new HookDeclaration(budgetUnit, true, true, false); + } + + /** The same, and a storage that can divide a copy in place under {@code LIN-053}. */ + public static HookDeclaration withLineage(String budgetUnit) { + return new HookDeclaration(budgetUnit, true, true, true); } } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java index 14bd055..56e9a67 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java @@ -41,6 +41,28 @@ public interface MovementHooks { /** Compensates an abort, under {@code MOVE-191}. */ HookResult rollback(HandoffContext context); + /** + * Divides the copy this node holds so that it matches the extent of {@code shardId}. + * + *

A local step under {@code LIN-051}: the node holds {@code sourceShardId} under the earlier + * snapshot and {@code shardId} under the later one, and nothing moves between nodes. The + * default refuses, because a storage that has not implemented it declares + * {@code supportsLineage} of false and {@code LIN-053} refuses the plan before a call is made. + */ + default HookResult divide(HandoffContext context) { + return new HookResult.Permanent("divide is not implemented"); + } + + /** + * Folds the copies this node holds into one that matches the extent of {@code shardId}. + * + *

The inverse of {@link #divide}, and what {@code LIN-056} calls to undo a division that was + * aborted, which is why a storage declares support for the pair rather than for either alone. + */ + default HookResult combine(HandoffContext context) { + return new HookResult.Permanent("combine is not implemented"); + } + /** Reads the durable state, under {@code MOVE-201}. */ ObserveResult observe(HandoffContext context); } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java index 95b34d5..cafba3d 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java @@ -1,6 +1,7 @@ package com.codeheadsystems.sharder.migrate.internal; import com.codeheadsystems.sharder.NodeId; +import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot; import com.codeheadsystems.sharder.error.InvalidArgumentException; @@ -41,8 +42,10 @@ public MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHo if (!target.engine().supportsOrchestratedMigration()) { throw new PlanRefusedException(PlanRefusedException.Cause.STRATEGY_UNSUPPORTED); } - var machine = com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.of( - source.engine(), target.engine(), policy.quiesceLeaseMarginMillis()); + // LIN-022: a boundary that neither divides nor folds an extent whole has no lineage to + // name, and LIN-013 refuses a directory pair whose shard sets differ until its extents are + // defined. Both surface here, before any handoff is admitted. + var machine = planOver(source, target, policy); machine.policy(policy.maxConcurrentHandoffs(), policy.maxConcurrentPerSourceNode(), policy.maxConcurrentPerDestinationNode()); // MOVE-333 and MOVE-336 both read the deadline, so the policy's value reaches the machine @@ -55,9 +58,32 @@ public MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHo PlanRefusedException.Cause.DESTINATION_OUTSIDE_PLACEMENT_SET); } } + // LIN-053: whether the storage can divide a copy in place is the integrator's statement, + // so a plan needing a local step is refused rather than calling a hook that is not there. + boolean needsLocalStep = machine.handoffs().stream() + .anyMatch(id -> machine.handoff(id).kind().local()); + if (needsLocalStep && !hooks.declare().supportsLineage()) { + throw new PlanRefusedException(PlanRefusedException.Cause.LINEAGE_UNSUPPORTED); + } return new DefaultMigrationPlan(machine, source, target, hooks, policy); } + /** The plan, with a lineage refusal reported as the condition {@code ERR-050} names. */ + private static com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan planOver( + DocumentSnapshot source, DocumentSnapshot target, MigrationPolicy policy) { + try { + return com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.of( + source.engine(), target.engine(), policy.quiesceLeaseMarginMillis()); + } catch (ShardExtents.Refused refusal) { + throw new PlanRefusedException(switch (refusal.cause()) { + case "unalignedLineage" -> PlanRefusedException.Cause.UNALIGNED_LINEAGE; + case "lineageUnsupported" -> PlanRefusedException.Cause.LINEAGE_UNSUPPORTED; + case "strategyUnsupported" -> PlanRefusedException.Cause.STRATEGY_UNSUPPORTED; + default -> PlanRefusedException.Cause.INCOMPARABLE_SHARDS; + }); + } + } + private static DocumentSnapshot snapshot(TopologySnapshot snapshot, String name) { if (snapshot instanceof DocumentSnapshot document) { return document; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java index 4a2e3af..50705f3 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java @@ -230,6 +230,7 @@ private StepOutcome advance(String id, Handoff handoff, Driving state, long at, HandoffState from = handoff.state(); return switch (from) { case PLANNED -> admit(id, handoff, level, at); + case DIVIDING -> divide(id, handoff, state, at); case PREPARING -> afterHook(id, state, at, from, hooks.prepare(context(handoff, state)), "prepareSuccess", "attemptsExhausted"); @@ -261,7 +262,25 @@ private StepOutcome admit(String id, Handoff handoff, PressureLevel level, long return null; } return new StepOutcome.Advanced(HandoffId.of(id), HandoffState.PLANNED, - HandoffState.PREPARING); + handoff.kind().local() ? HandoffState.DIVIDING : HandoffState.PREPARING); + } + + /** + * {@code LIN-052}: the local step, which divides a copy or folds several into one. + * + *

Nothing moves between nodes, so there is no destination to prepare and no cutover to + * commit. A permanent answer is the one case that reaches {@code failed} directly, with the + * kind {@code undivided} of {@code MOVE-011}, because the copy matches neither extent. + */ + private StepOutcome divide(String id, Handoff handoff, Driving state, long at) { + boolean folding = handoff.kind() == Handoff.Kind.COMBINE; + HandoffContext context = context(handoff, state); + HookResult answer = folding ? hooks.combine(context) : hooks.divide(context); + if (answer instanceof HookResult.Permanent) { + return advanced(id, HandoffState.DIVIDING, stepMachine(id, "dividePermanent", at)); + } + return afterHook(id, state, at, HandoffState.DIVIDING, answer, + folding ? "combineSuccess" : "divideSuccess", "attemptsExhausted"); } private StepOutcome transfer(String id, Handoff handoff, Driving state, long at) { @@ -465,9 +484,9 @@ private StepOutcome advanced(String id, /** The context one hook call reads, under {@code MOVE-111}. */ private HandoffContext context(Handoff handoff, Driving state) { - return new HandoffContext(ShardId.of(handoff.shard()), source.topologyId(), - handoff.fromEpoch(), handoff.toEpoch(), handoff.source(), handoff.destination(), - state.attempts + 1, policy.stepDeadlineMillis()); + return new HandoffContext(ShardId.of(handoff.shard()), ShardId.of(handoff.sourceShard()), + source.topologyId(), handoff.fromEpoch(), handoff.toEpoch(), handoff.source(), + handoff.destination(), state.attempts + 1, policy.stepDeadlineMillis()); } @Override diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java b/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java index 0833e65..cbe1b82 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java @@ -171,7 +171,7 @@ void aMismatchedVerificationFailsTheHandoff() { void hooksThatDeclareNoVerificationReachCleanupFromTheRecord() { Pair pair = snapshots(slots(1, "a", "b"), slots(2, "a", "c")); RecordingHooks hooks = new RecordingHooks() - .declaring(new HookDeclaration("rows", true, false)); + .declaring(new HookDeclaration("rows", true, false, false)); MigrationPlan plan = Sharder.coordinator() .plan(pair.from(), pair.to(), hooks, MigrationPolicy.defaults()); for (int step = 0; step < 12; step++) { diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java index 1a134e4..3561c3a 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java @@ -342,15 +342,23 @@ void planConstruction(JsonObject testCase) { assertThat(handoff).as("handoffs[%d] named %s", index, entry.text("id")).isNotNull(); assertThat(handoff.shard()).as("handoffs[%d].shard", index) .isEqualTo(entry.text("shard")); + assertThat(handoff.kind().spelling()).as("handoffs[%d].kind", index) + .isEqualTo(entry.text("kind")); assertThat(handoff.sourceShard()).as("handoffs[%d].sourceShard", index) .isEqualTo(entry.text("sourceShard")); assertThat(handoff.source().asText()).as("handoffs[%d].source", index) .isEqualTo(entry.text("source")); assertThat(handoff.destination().asText()).as("handoffs[%d].destination", index) .isEqualTo(entry.text("destination")); - // LIN-042: a node cannot both hold the contents and be the node they move to. - assertThat(handoff.source()).as("handoffs[%d] source is not the destination", index) - .isNotEqualTo(handoff.destination()); + if (handoff.kind().local()) { + // LIN-051: a local step moves nothing between nodes, so it names one node twice. + assertThat(handoff.source()).as("handoffs[%d] is local", index) + .isEqualTo(handoff.destination()); + } else { + // LIN-042: a node cannot both hold the contents and be the node they move to. + assertThat(handoff.source()).as("handoffs[%d] source is not the destination", index) + .isNotEqualTo(handoff.destination()); + } } } diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java index 9390b92..567f441 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java @@ -62,7 +62,19 @@ int run(String path) { new java.util.ArrayList<>(); for (JsonValue row : spec.array("handoffs").elements()) { JsonObject entry = row.asObject(); - named.add(MigrationPlan.handoffOf(entry.text("id"), entry.text("shard"), + String shard = entry.text("shard"); + String sourceShard = entry.find("sourceShard") + .map(JsonValue::asText).orElse(shard); + com.codeheadsystems.sharder.core.internal.migrate.Handoff.Kind kind = + switch (entry.find("kind").map(JsonValue::asText).orElse("handoff")) { + case "divide" -> com.codeheadsystems.sharder.core.internal.migrate + .Handoff.Kind.DIVIDE; + case "combine" -> com.codeheadsystems.sharder.core.internal.migrate + .Handoff.Kind.COMBINE; + default -> com.codeheadsystems.sharder.core.internal.migrate + .Handoff.Kind.HANDOFF; + }; + named.add(MigrationPlan.handoffOf(entry.text("id"), shard, sourceShard, kind, NodeId.of(entry.text("source")), NodeId.of(entry.text("destination")), spec.get("fromEpoch").asLong(), spec.get("toEpoch").asLong())); } From 000b4c2f465fba6ec9c8d674111af33f5a7d245b Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 08:33:07 -0700 Subject: [PATCH 4/7] Decide a directory extent over the prefix trie `LIN-013` refused a `directory` pair whose entry sets differ, which was a correct answer standing in for the right one while the ring geometry settled. It is restated here to define the extent, and the refusal goes with it. A directory extent is a matcher narrowed by the precedence of `PLACE-065`: the keys its entry wins once every entry of the same table that outranks it has taken what it claims. An exact matcher outranks every prefix and a longer prefix outranks a shorter one, so every entry that outranks another and overlaps it is contained in it, and an extent is one matcher minus a finite set of matchers inside it. `LIN-016` states how that is decided without evaluating the table over generated keys, which `LIN-001` forbids for the same reason `TOPO-221` forbids it of the ownership delta. Every decoded matcher value of either table is a node of one trie, and each node contributes two regions: the key equal to it, and the keys strictly extending it that no deeper node is a prefix of. Two keys of one region match exactly the same entries of either table, so a region is the finest distinction either table can draw and the regions together admit every key. An extent is the set of regions its entry wins, and equality, containment, and disjointness are set operations over those. An exact matcher wins only the region that is its own key. A region of keys strictly extending a node contains no node of the trie, and every matcher value is one, so no exact matcher can lie inside it. Refining `prefix:ab` into `ab0` and `ab1` while keeping `ab` divides one extent into three, because `ab` still wins the keys neither longer prefix claims. An entry that wins keys the earlier table matched to no shard under `DIR-010` names a shard with no parent, which is the `fresh` class, and `LIN-043` emits no handoff for it. That case exists only under `directory`, because it is the only kind whose `shardOf` answers with no shard, and it is why the requirement had no executable test until now. Every port declaring `migration` and exposing `directory` implements this, which is the scoping the strategy surface axis already gives and 0087 argues for. Co-Authored-By: Claude Opus 5 --- conformance/coverage.json | 28 ++- conformance/declarations/java.json | 8 +- conformance/generator/generate_extra.py | 34 +++- conformance/generator/sharder_ref/lineage.py | 117 ++++++++++-- conformance/manifest.json | 33 +++- .../lineage-directory-fresh.topology.json | 48 +++++ conformance/vectors/migration/lineage.json | 104 +++++++++-- docs/design/10-specification.md | 18 +- docs/design/30-conformance.md | 5 +- ports/java/USER-GUIDE.md | 7 +- ports/java/conformance/report.txt | 18 +- .../core/internal/placement/ShardExtents.java | 173 +++++++++++++++--- 12 files changed, 485 insertions(+), 108 deletions(-) create mode 100644 conformance/topologies/lineage-directory-fresh.topology.json diff --git a/conformance/coverage.json b/conformance/coverage.json index 72e385c..ae97bab 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -1,7 +1,7 @@ { "specification": "docs/design/10-specification.md", - "statedRequirements": 688, - "coveredRequirements": 472, + "statedRequirements": 689, + "coveredRequirements": 473, "uncoveredRequirements": 216, "coveragePercent": 68, "identifiersNamedButNotStated": [], @@ -9,8 +9,8 @@ { "prefix": "LIN", "section": "", - "stated": 32, - "covered": 18, + "stated": 33, + "covered": 19, "uncovered": [ "LIN-001", "LIN-002", @@ -27,7 +27,7 @@ "LIN-055", "LIN-058" ], - "percent": 56 + "percent": 57 }, { "prefix": "CFG", @@ -981,6 +981,8 @@ "CFG-050", "CFG-052", "CFG-055", + "DIR-002", + "DIR-010", "ERR-050", "ERR-052", "ERR-053", @@ -995,6 +997,7 @@ "LIN-012", "LIN-013", "LIN-014", + "LIN-016", "LIN-021", "LIN-022", "LIN-033", @@ -1062,6 +1065,7 @@ "OBS-010", "OBS-020", "OBS-021", + "PLACE-065", "RATE-001", "RATE-011", "RATE-021", @@ -1077,8 +1081,8 @@ "TOPO-221", "TOPO-231" ], - "requirementsAtLevel": 98, - "requirementsWhenDeclared": 422 + "requirementsAtLevel": 102, + "requirementsWhenDeclared": 423 } ], "coveredBy": { @@ -1320,7 +1324,8 @@ "vectors/directory/matcher-precedence.json" ], "DIR-002": [ - "vectors/directory/matcher-precedence.json" + "vectors/directory/matcher-precedence.json", + "vectors/migration/lineage.json" ], "DIR-003": [ "vectors/directory/matcher-precedence.json", @@ -1358,11 +1363,13 @@ ], "PLACE-065": [ "vectors/directory/matcher-precedence.json", + "vectors/migration/lineage.json", "vectors/overrides/precedence-and-composition.json", "vectors/rendezvous/tenant-router.json" ], "DIR-010": [ - "vectors/directory/no-match.json" + "vectors/directory/no-match.json", + "vectors/migration/lineage.json" ], "DIR-011": [ "vectors/directory/no-match.json" @@ -1696,6 +1703,9 @@ "LIN-014": [ "vectors/migration/lineage.json" ], + "LIN-016": [ + "vectors/migration/lineage.json" + ], "LIN-021": [ "vectors/migration/lineage.json" ], diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index 36e5c8d..4af016a 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "0cbfa051871dc2e6467163cc79a276dc19769add35ba8e5c5fd14e5b6cab2312", + "revision": "e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49", "strategySurfaces": [ "directory", "rendezvous", @@ -38,12 +38,12 @@ "command": "cd ports/java && ./gradlew test", "report": "ports/java/conformance/report.txt", "vectorFiles": 69, - "vectorCases": 608, + "vectorCases": 609, "failures": 0 }, "scale": { - "wallTimeMs": 892, - "peakResidentBytes": 602157056, + "wallTimeMs": 873, + "peakResidentBytes": 601296896, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/generator/generate_extra.py b/conformance/generator/generate_extra.py index fe39c61..d0e7d25 100644 --- a/conformance/generator/generate_extra.py +++ b/conformance/generator/generate_extra.py @@ -425,6 +425,17 @@ def build_identity_comparison(root): "nodes": [{"id": "d1"}, {"id": "d2"}], } +# `LIN-043`: `ef` matched no entry of the earlier table, so the shard it names has no parent and +# no contents to move. A `directory` table is the only place a fresh extent arises, because it is +# the only kind whose `shardOf` answers with no shard under `DIR-010`. +LINEAGE_DIR_FRESH = copy.deepcopy(LINEAGE_DIR_BEFORE) +LINEAGE_DIR_FRESH["epoch"] = 3 +LINEAGE_DIR_FRESH["strategy"]["entries"] = [ + {"match": {"kind": "prefix", "value": "ab"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "cd"}, "nodes": ["d2"]}, + {"match": {"kind": "prefix", "value": "ef"}, "nodes": ["d1"]}, +] + LINEAGE_DIR_REFINED = copy.deepcopy(LINEAGE_DIR_BEFORE) LINEAGE_DIR_REFINED["epoch"] = 2 LINEAGE_DIR_REFINED["strategy"]["entries"] = [ @@ -439,6 +450,7 @@ def build_lineage(root): """`LIN-*`: the classification over pairs of documents, and the plan built over it.""" documents = {"lineage-directory-before": LINEAGE_DIR_BEFORE, "lineage-directory-refined": LINEAGE_DIR_REFINED, + "lineage-directory-fresh": LINEAGE_DIR_FRESH, "delta-ring-unaligned": DELTA_RING_UNALIGNED} for name, document in documents.items(): write_json(root / ("topologies/%s.topology.json" % name), document) @@ -470,6 +482,15 @@ def path(name): "`LIN-021`: removing the token folds `(1000, 2000]` into the extent that follows it, so " "the later shard is `merged` from two parents and the shard only the earlier snapshot " "enumerates is `folded` and follows every later entry under `LIN-033`."), + ("directory-prefix-refined", "lineage-directory-before", "lineage-directory-refined", + ["LIN-013", "LIN-016", "LIN-021", "DIR-002", "PLACE-065"], + "`LIN-016`: refining `prefix:ab` into `ab0` and `ab1` leaves `ab` winning the keys " + "neither longer prefix claims, so all three shards of the later table are `divided` from " + "the one entry and the untouched `cd` is `unchanged`."), + ("directory-fresh-extent", "lineage-directory-before", "lineage-directory-fresh", + ["LIN-013", "LIN-016", "LIN-021", "DIR-010"], + "`LIN-021`: the added entry wins keys the earlier table matched to no shard under " + "`DIR-010`, so the shard it names is `fresh` and has no parent to draw from."), ("slot-identity", "delta-before", "delta-moved", ["LIN-007", "LIN-012", "LIN-021"], "`LIN-012`: `TOPO-231` holds `slotCount` equal, so the two snapshots enumerate the same " @@ -488,10 +509,6 @@ def path(name): "`LIN-022`: a boundary that moves without dividing or folding an extent whole has no " "correspondence to name, so the plan is refused and the authority publishes the change " "as a division epoch followed by a fold epoch."), - ("directory-differing-shard-sets", "lineage-directory-before", "lineage-directory-refined", - ["LIN-013", "ERR-050"], "unalignedLineage", - "`LIN-013`: a `directory` extent is a matcher narrowed by the precedence of `DIR-002`. " - "It is decidable and not yet defined, so a pair whose shard sets differ is refused."), ("incomparable-shard-identity", "delta-before", "delta-other-seed", ["LIN-006", "TOPO-231", "ERR-050"], "incomparableShards", "`LIN-006`: a lineage joins two snapshots on their extents and an ownership delta joins " @@ -512,10 +529,11 @@ def path(name): emit(root, "vectors/migration/lineage.json", "migration-lineage", "lineage", "The lineage classification over pairs of documents: a ring extent divided, a ring extent " "folded, the identity lineage under `slot`, the unaligned boundary that is refused, the " - "`directory` pair whose shard sets differ, the incomparable pair, and the kind that " - "enumerates no shard.", - ["LIN-004", "LIN-006", "LIN-007", "LIN-011", "LIN-012", "LIN-013", "LIN-014", "LIN-021", - "LIN-022", "LIN-033", "TOPO-231", "MOVE-241", "ERR-050"], cases, level="migration") + "`directory` prefix refined and the fresh extent beside it, the incomparable pair, and " + "the kind that enumerates no shard.", + ["LIN-004", "LIN-006", "LIN-007", "LIN-011", "LIN-012", "LIN-013", "LIN-014", "LIN-016", + "LIN-021", "LIN-022", "LIN-033", "DIR-002", "DIR-010", "PLACE-065", "TOPO-231", + "MOVE-241", "ERR-050"], cases, level="migration") plans = [] for label, before, after, requirements, note in [ diff --git a/conformance/generator/sharder_ref/lineage.py b/conformance/generator/sharder_ref/lineage.py index bc9c621..f6b85b4 100644 --- a/conformance/generator/sharder_ref/lineage.py +++ b/conformance/generator/sharder_ref/lineage.py @@ -77,6 +77,80 @@ def equal(first, second): return _measure(first) == _measure(second) == _measure(_intersection(first, second)) +# ------------------------------------------------------- extents under `directory` + +def _matchers(snapshot): + """Each entry's matcher as `(kind, decoded octets)`, in `entries` array order.""" + from .topology import decode_matcher_value + + out = [] + for entry in snapshot.strategy["entries"]: + match = entry["match"] + out.append((match["kind"], decode_matcher_value(match))) + return out + + +def _regions(first, second): + """The regions the two tables' matcher values cut the keyspace into. + + Every node of the combined prefix trie contributes two regions: the key equal to that node, and + the keys strictly extending it that no deeper node is a prefix of. The root contributes the + second of those for the keys no matcher value is a prefix of. Two keys in one region match + exactly the same matchers of either table, so a region is the finest distinction either table + can draw and the regions together cover the keyspace. + """ + nodes = {b""} + for kind, value in list(first) + list(second): + nodes.add(value) + ordered = sorted(nodes) + return ([("exact", node) for node in ordered if node] + + [("open", node) for node in ordered]) + + +def _winner(matchers, region): + """`PLACE-065`: the entry index that wins a region, or `None` where no entry matches it. + + An `exact` matcher can only win the region that is its own key, because a region of keys + strictly extending a node contains no node of the trie and every matcher value is one. + """ + shape, node = region + if shape == "exact": + for index, (kind, value) in enumerate(matchers): + if kind == "exact" and value == node: + return index + best = None + for index, (kind, value) in enumerate(matchers): + if kind != "prefix" or not node.startswith(value): + continue + if best is None or len(value) > len(matchers[best][1]): + best = index + return best + + +def directory_extents(before, after): + """`LIN-013`: each shard's extent as the set of regions its entry wins.""" + first, second = _matchers(before), _matchers(after) + regions = _regions(first, second) + out = [] + for snapshot, matchers in ((before, first), (after, second)): + shards = placement.shards(snapshot) + extents = {shard: set() for shard in shards} + for region in regions: + index = _winner(matchers, region) + if index is not None: + extents[shards[index]].add(region) + out.append({shard: frozenset(region) for shard, region in extents.items()}) + return out[0], out[1] + + +def _region_meets(first, second): + return bool(first & second) + + +def _region_contains(outer, inner): + return inner <= outer + + # ------------------------------------------------------------------------------- the lineage def extents(snapshot): @@ -134,31 +208,28 @@ def classify(before, after, replicas_of): raise Refused("strategyUnsupported", "rendezvous enumerates no shard") shards_before, shards_after = placement.shards(before), placement.shards(after) - if kind in ("slot", "directory"): - if shards_before != shards_after: - # `LIN-013`: a directory extent is decidable and not yet defined, so the pair is - # refused rather than guessed. Under `slot` `TOPO-231` has already refused a differing - # `slotCount`, so an unequal shard set cannot arise. - raise Refused("unalignedLineage", "the two snapshots enumerate different shards") + if kind == "slot": + # `LIN-012`: `TOPO-231` has already refused a differing `slotCount`, so the two snapshots + # enumerate the same slots and every extent equals its counterpart. return _identity_lineage(before, after, replicas_of) # `LIN-007`: an equal shard set is an equal extent set, so the lineage is the identity. if shards_before == shards_after: return _identity_lineage(before, after, replicas_of) - earlier, later = extents(before), extents(after) + earlier, later, meets_, contains_, equal_ = _geometry(before, after, kind) rows = [] for shard in shards_after: mine = later[shard] - parents = [s for s in shards_before if meets(earlier[s], mine)] + parents = [s for s in shards_before if meets_(earlier[s], mine)] for parent in parents: - if not contains(earlier[parent], mine) and not contains(mine, earlier[parent]): + if not contains_(earlier[parent], mine) and not contains_(mine, earlier[parent]): raise Refused("unalignedLineage", shard) if not parents: rows.append({"shard": shard, "class": "fresh", "parents": []}) elif len(parents) > 1: rows.append({"shard": shard, "class": "merged", "parents": parents}) - elif equal(earlier[parents[0]], mine): + elif equal_(earlier[parents[0]], mine): was, now = replicas_of(before, shard), replicas_of(after, shard) rows.append({"shard": shard, "class": "unchanged" if was == now else "moved", "parents": parents}) @@ -169,7 +240,7 @@ def classify(before, after, replicas_of): if shard in later: continue mine = earlier[shard] - children = [s for s in shards_after if meets(later[s], mine)] + children = [s for s in shards_after if meets_(later[s], mine)] if not children: rows.append({"shard": shard, "class": "vacated", "parents": []}) elif len(children) > 1: @@ -179,14 +250,22 @@ def classify(before, after, replicas_of): return rows +def _geometry(before, after, kind): + """The two extent maps and the three predicates that decide them, per strategy kind.""" + if kind == "directory": + earlier, later = directory_extents(before, after) + return earlier, later, _region_meets, _region_contains, lambda a, b: a == b + return ring_extents(before), ring_extents(after), meets, contains, equal + + def parents_of(before, after): """For each shard of the later snapshot, the shards of the earlier one its extent draws from.""" kind = before.strategy["kind"] shards_before, shards_after = placement.shards(before), placement.shards(after) - if kind in ("slot", "directory") or shards_before == shards_after: + if kind == "slot" or shards_before == shards_after: return {shard: [shard] for shard in shards_after} - earlier, later = extents(before), extents(after) - return {shard: [s for s in shards_before if meets(earlier[s], later[shard])] + earlier, later, meets_, _, _ = _geometry(before, after, kind) + return {shard: [s for s in shards_before if meets_(earlier[s], later[shard])] for shard in shards_after} @@ -198,16 +277,18 @@ def plan_handoffs(before, after, replicas_of): """ parents = parents_of(before, after) kind = before.strategy["kind"] - identity = kind != "ring" or placement.shards(before) == placement.shards(after) - earlier = {} if identity else extents(before) - later = {} if identity else extents(after) + identity = kind == "slot" or placement.shards(before) == placement.shards(after) + if identity: + earlier, later, equal_ = {}, {}, None + else: + earlier, later, _, _, equal_ = _geometry(before, after, kind) out = [] for shard in placement.shards(after): destinations = replicas_of(after, shard) mine = parents[shard] divisions, folds, moves = [], [], [] - if not identity and len(mine) == 1 and not equal(earlier[mine[0]], later[shard]): + if not identity and len(mine) == 1 and not equal_(earlier[mine[0]], later[shard]): # `LIN-052`: the extent narrowed, so a node holding both divides its own copy. for node in destinations: if node in replicas_of(before, mine[0]): diff --git a/conformance/manifest.json b/conformance/manifest.json index f4ddc25..db2be58 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,9 +1,9 @@ { "suite": "sharder conformance suite", "revision": { - "id": "0cbfa051871dc2e6467163cc79a276dc19769add35ba8e5c5fd14e5b6cab2312", + "id": "e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", - "fileCount": 204, + "fileCount": 205, "trees": [ "vectors", "topologies", @@ -16,12 +16,12 @@ "generator": "conformance/generator", "counts": { "vectorFiles": 69, - "vectorCases": 608, - "topologyDocuments": 108, + "vectorCases": 609, + "topologyDocuments": 109, "scenarios": 25, "scenarioSteps": 313, "properties": 28, - "requirementsNamed": 472 + "requirementsNamed": 473 }, "requirementsNamed": [ "CFG-014", @@ -196,6 +196,7 @@ "LIN-012", "LIN-013", "LIN-014", + "LIN-016", "LIN-021", "LIN-022", "LIN-033", @@ -587,10 +588,10 @@ ], "surface": "the handoff coordinator and rate control", "vectorFiles": 5, - "vectorCases": 25, + "vectorCases": 26, "scenarios": 15, "properties": 3, - "requirementsNamed": 98 + "requirementsNamed": 102 } ], "strategySurfaces": [ @@ -1065,6 +1066,14 @@ "digest": "202c08ccc95f89693fff2411d987ea83bf6e54536b0ff0434d7b77027cf2588d", "valid": true }, + "topologies/lineage-directory-fresh.topology.json": { + "topologyId": "lineage-directory", + "epoch": 3, + "strategy": "directory", + "nodeCount": 2, + "digest": "3ef0ed21f04db339d03aac9c12d7d6693578cb72df336c453716e77030559697", + "valid": true + }, "topologies/lineage-directory-refined.topology.json": { "topologyId": "lineage-directory", "epoch": 2, @@ -2054,10 +2063,12 @@ "ring", "slot" ], - "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` pair whose shard sets differ, the incomparable pair, and the kind that enumerates no shard.", + "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` prefix refined and the fresh extent beside it, the incomparable pair, and the kind that enumerates no shard.", "topology": null, - "caseCount": 7, + "caseCount": 8, "requirements": [ + "DIR-002", + "DIR-010", "ERR-050", "LIN-004", "LIN-006", @@ -2066,13 +2077,15 @@ "LIN-012", "LIN-013", "LIN-014", + "LIN-016", "LIN-021", "LIN-022", "LIN-033", "MOVE-241", + "PLACE-065", "TOPO-231" ], - "sha256": "39281cb0f3216695062075b0935328864baf6aede3a19b3df5033c36e10a1182" + "sha256": "1e13da445183131e15811f7b28046afd0094a7bc21692f6ecf2aff6e0007e9b2" }, { "file": "vectors/migration/plan-construction.json", diff --git a/conformance/topologies/lineage-directory-fresh.topology.json b/conformance/topologies/lineage-directory-fresh.topology.json new file mode 100644 index 0000000..b60b25d --- /dev/null +++ b/conformance/topologies/lineage-directory-fresh.topology.json @@ -0,0 +1,48 @@ +{ + "formatVersion": "1.0", + "topologyId": "lineage-directory", + "epoch": 3, + "replication": { + "factor": 1 + }, + "strategy": { + "kind": "directory", + "entries": [ + { + "match": { + "kind": "prefix", + "value": "ab" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "cd" + }, + "nodes": [ + "d2" + ] + }, + { + "match": { + "kind": "prefix", + "value": "ef" + }, + "nodes": [ + "d1" + ] + } + ] + }, + "nodes": [ + { + "id": "d1" + }, + { + "id": "d2" + } + ] +} diff --git a/conformance/vectors/migration/lineage.json b/conformance/vectors/migration/lineage.json index 2e15a74..8f7e51e 100644 --- a/conformance/vectors/migration/lineage.json +++ b/conformance/vectors/migration/lineage.json @@ -2,8 +2,10 @@ "vectorSet": "migration-lineage", "kind": "lineage", "level": "migration", - "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` pair whose shard sets differ, the incomparable pair, and the kind that enumerates no shard.", + "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` prefix refined and the fresh extent beside it, the incomparable pair, and the kind that enumerates no shard.", "requirements": [ + "DIR-002", + "DIR-010", "ERR-050", "LIN-004", "LIN-006", @@ -12,10 +14,12 @@ "LIN-012", "LIN-013", "LIN-014", + "LIN-016", "LIN-021", "LIN-022", "LIN-033", "MOVE-241", + "PLACE-065", "TOPO-231" ], "cases": [ @@ -122,6 +126,86 @@ ] } }, + { + "name": "directory-prefix-refined", + "requirements": [ + "LIN-013", + "LIN-016", + "LIN-021", + "DIR-002", + "PLACE-065" + ], + "before": "topologies/lineage-directory-before.topology.json", + "after": "topologies/lineage-directory-refined.topology.json", + "note": "`LIN-016`: refining `prefix:ab` into `ab0` and `ab1` leaves `ab` winning the keys neither longer prefix claims, so all three shards of the later table are `divided` from the one entry and the untouched `cd` is `unchanged`.", + "expect": { + "lineage": [ + { + "shard": "prefix:616230", + "class": "divided", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:616231", + "class": "divided", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:6162", + "class": "divided", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:6364", + "class": "unchanged", + "parents": [ + "prefix:6364" + ] + } + ] + } + }, + { + "name": "directory-fresh-extent", + "requirements": [ + "LIN-013", + "LIN-016", + "LIN-021", + "DIR-010" + ], + "before": "topologies/lineage-directory-before.topology.json", + "after": "topologies/lineage-directory-fresh.topology.json", + "note": "`LIN-021`: the added entry wins keys the earlier table matched to no shard under `DIR-010`, so the shard it names is `fresh` and has no parent to draw from.", + "expect": { + "lineage": [ + { + "shard": "prefix:6162", + "class": "unchanged", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:6364", + "class": "unchanged", + "parents": [ + "prefix:6364" + ] + }, + { + "shard": "prefix:6566", + "class": "fresh", + "parents": [] + } + ] + } + }, { "name": "slot-identity", "requirements": [ @@ -198,24 +282,6 @@ } } }, - { - "name": "directory-differing-shard-sets", - "requirements": [ - "LIN-013", - "ERR-050" - ], - "before": "topologies/lineage-directory-before.topology.json", - "after": "topologies/lineage-directory-refined.topology.json", - "note": "`LIN-013`: a `directory` extent is a matcher narrowed by the precedence of `DIR-002`. It is decidable and not yet defined, so a pair whose shard sets differ is refused.", - "expect": { - "lineageComputed": false, - "condition": { - "code": 401, - "name": "planRefused", - "cause": "unalignedLineage" - } - } - }, { "name": "incomparable-shard-identity", "requirements": [ diff --git a/docs/design/10-specification.md b/docs/design/10-specification.md index 8cd2f23..d5bb067 100644 --- a/docs/design/10-specification.md +++ b/docs/design/10-specification.md @@ -2978,10 +2978,20 @@ owning ring entry. Containment and equality MUST be decided by comparing interva enumerate the same slots and every extent is equal to its counterpart. The lineage under `slot` MUST therefore be the identity of `LIN-007`. -`LIN-013`. Under `directory`, an implementation MUST refuse to compute a lineage where the two -snapshots enumerate different shard sets, and MUST report the refusal under `ERR-050` with the cause -`unalignedLineage`. A directory extent is a matcher narrowed by the precedence of `DIR-002`, so it -is decidable, and this requirement states the refusal that stands until it is defined. +`LIN-013`. Under `directory`, the extent of a shard MUST be the keys its entry wins under the +precedence of `PLACE-065`, which is its matcher narrowed by every entry of the same table that +outranks it. An implementation MUST decide equality, containment, and disjointness of two directory +extents over the prefix trie of the two tables' decoded matcher values, and MUST NOT decide them by +evaluating the tables over generated keys. + +`LIN-016`. The trie of `LIN-013` MUST be read as follows. Every decoded matcher value of either +table is a node, and the empty value is a node. Each node contributes two regions of the keyspace: +the key equal to that node, and the keys strictly extending it that no deeper node is a prefix of. +Two keys of one region match the same entries of either table, so a region is the finest distinction +either table draws, and the regions together admit every key. The extent of a shard is the set of +regions its entry wins, and an implementation MUST compare two extents as those sets. An `exact` +matcher wins only the region that is its own key, because a region of keys strictly extending a node +contains no node and every matcher value is one. `LIN-014`. Under `rendezvous`, `PLACE-032` makes `shards` empty and `MOVE-241` excludes the kind from orchestrated migration, so there is no extent and no lineage. An implementation MUST refuse a diff --git a/docs/design/30-conformance.md b/docs/design/30-conformance.md index ee232a0..7c88047 100644 --- a/docs/design/30-conformance.md +++ b/docs/design/30-conformance.md @@ -777,10 +777,9 @@ describes. The state sequence a local step runs is an output, so `scenarios/handoff-local-division.json`, `scenarios/handoff-local-fold.json`, and `scenarios/handoff-local-step-aborted.json` carry `LIN-051`, `LIN-052`, `LIN-056`, and `LIN-057`. -`LIN-043` states that a plan emits no handoff for a shard with no parent. A fresh extent arises only +`LIN-043` states that a plan emits no handoff for a shard with no parent. A fresh extent arises where a later snapshot admits routing keys that the earlier one matched to no shard, which is the -`directory` no-match of `DIR-010`, and `LIN-013` refuses a `directory` pair whose shard sets differ -until directory extents are defined. The case arrives with them. +`directory` no-match of `DIR-010`, and `vectors/migration/lineage.json` carries that case. ### Configuration and security diff --git a/ports/java/USER-GUIDE.md b/ports/java/USER-GUIDE.md index 22a2954..7f53938 100644 --- a/ports/java/USER-GUIDE.md +++ b/ports/java/USER-GUIDE.md @@ -593,9 +593,10 @@ Publish it as two epochs instead: the first divides every extent the change cros folds the pieces into their destinations. Each is plannable on its own, and each leaves a topology that routes correctly if the second is delayed. -Under `directory` a pair whose entry sets differ is refused for now with the same cause, until -directory extents are defined. A directory topology whose entries are unchanged plans as it always -did. +Under `directory` an extent is a matcher narrowed by the entries that outrank it, so refining +`prefix:ab` into `ab0` and `ab1` is a division and dropping the two back to `ab` is a fold. An entry +that wins keys the earlier table matched to nothing names a shard with no parent, which moves no +contents and needs no handoff. ### A new epoch diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index b170649..dba7d73 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,17 +1,17 @@ sharder conformance report, Java port -suite revision 0cbfa051871dc2e6467163cc79a276dc19769add35ba8e5c5fd14e5b6cab2312 +suite revision e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49 hash 2 of 2 vector files, 34 of 34 cases, 0.01s - place 45 of 45 vector files, 291 of 291 cases, 0.09s - core 8 of 8 vector files, 135 of 135 cases, 8.07s - scale 2 of 2 vector files, 2 of 2 cases, 0.89s + place 45 of 45 vector files, 291 of 291 cases, 0.10s + core 8 of 8 vector files, 135 of 135 cases, 8.27s + scale 2 of 2 vector files, 2 of 2 cases, 0.87s failover 3 of 3 vector files, 67 of 67 cases, 0.00s readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s fencing 3 of 3 vector files, 8 of 8 cases, 0.00s - migration 5 of 5 vector files, 25 of 25 cases, 0.01s + migration 5 of 5 vector files, 26 of 26 cases, 0.01s - 608 cases, 0 failures - scale wall time 0.89s - scale wall time 892 ms - peak resident size 602157056 bytes (574 MiB) + 609 cases, 0 failures + scale wall time 0.87s + scale wall time 873 ms + peak resident size 601296896 bytes (573 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java index 0e517d9..5d64103 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java @@ -1,9 +1,11 @@ package com.codeheadsystems.sharder.core.internal.placement; +import com.codeheadsystems.sharder.core.internal.document.TopologyDocument; import com.codeheadsystems.sharder.core.internal.hash.U64; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; import java.util.ArrayList; +import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -147,6 +149,140 @@ private static boolean equal(List first, List second) { return contains(first, second) && contains(second, first); } + /** + * One region of the keyspace the two directory tables cut it into, under {@code LIN-016}. + * + *

{@code exact} is the key equal to {@code node}; otherwise it is the keys strictly + * extending {@code node} that no deeper node of the combined trie is a prefix of. + */ + private record Region(boolean exact, String node) { + } + + /** Every decoded matcher value of either table, rendered as hexadecimal, and the empty one. */ + private static List regions(TopologyDocument before, TopologyDocument after) { + Set nodes = new java.util.TreeSet<>(); + nodes.add(""); + for (TopologyDocument document : List.of(before, after)) { + for (TopologyDocument.DirectoryEntry entry : document.strategy().entries()) { + nodes.add(HexFormat.of().formatHex(entry.match().value())); + } + } + List out = new ArrayList<>(); + nodes.stream().filter(node -> !node.isEmpty()) + .forEach(node -> out.add(new Region(true, node))); + nodes.forEach(node -> out.add(new Region(false, node))); + return List.copyOf(out); + } + + /** + * {@code PLACE-065}: the entry index winning a region, or {@code -1} where none matches it. + * + *

An {@code exact} matcher wins only the region that is its own key, because a region of + * keys strictly extending a node contains no node and every matcher value is one. + */ + private static int winner(TopologyDocument document, Region region) { + List entries = document.strategy().entries(); + HexFormat hex = HexFormat.of(); + if (region.exact()) { + for (int index = 0; index < entries.size(); index++) { + TopologyDocument.Matcher matcher = entries.get(index).match(); + if ("exact".equals(matcher.kind()) + && hex.formatHex(matcher.value()).equals(region.node())) { + return index; + } + } + } + int best = -1; + int bestLength = -1; + for (int index = 0; index < entries.size(); index++) { + TopologyDocument.Matcher matcher = entries.get(index).match(); + if (!"prefix".equals(matcher.kind())) { + continue; + } + String value = hex.formatHex(matcher.value()); + if (!region.node().startsWith(value)) { + continue; + } + if (value.length() > bestLength) { + best = index; + bestLength = value.length(); + } + } + return best; + } + + /** {@code LIN-013}: each shard's extent as the set of regions its entry wins. */ + private static Map> directoryExtents(PlacementEngine engine, + List regions) { + List shards = engine.placement().shards(); + Map> out = new LinkedHashMap<>(); + shards.forEach(shard -> out.put(shard, new LinkedHashSet<>())); + for (Region region : regions) { + int index = winner(engine.document(), region); + if (index >= 0) { + out.get(shards.get(index)).add(region); + } + } + return out; + } + + /** + * A shard's extent, whichever geometry decides it. + * + *

Under {@code ring} it is a set of intervals of the hash space; under {@code directory} it + * is a set of regions of the prefix trie. The three predicates below are what the + * classification reads, so nothing above this point depends on which kind is in play. + */ + private sealed interface Extent permits RingExtent, DirExtent { + } + + private record RingExtent(List segments) implements Extent { + } + + private record DirExtent(Set regions) implements Extent { + } + + private static boolean meetsExtent(Extent first, Extent second) { + if (first instanceof RingExtent left && second instanceof RingExtent right) { + return meets(left.segments(), right.segments()); + } + Set left = ((DirExtent) first).regions(); + Set right = ((DirExtent) second).regions(); + return left.stream().anyMatch(right::contains); + } + + private static boolean containsExtent(Extent outer, Extent inner) { + if (outer instanceof RingExtent left && inner instanceof RingExtent right) { + return contains(left.segments(), right.segments()); + } + return ((DirExtent) outer).regions().containsAll(((DirExtent) inner).regions()); + } + + private static boolean equalExtent(Extent first, Extent second) { + if (first instanceof RingExtent left && second instanceof RingExtent right) { + return equal(left.segments(), right.segments()); + } + return ((DirExtent) first).regions().equals(((DirExtent) second).regions()); + } + + /** The two extent maps, decided by the geometry the strategy kind gives. */ + private static List> geometry(PlacementEngine before, + PlacementEngine after) { + Map earlier = new LinkedHashMap<>(); + Map later = new LinkedHashMap<>(); + if ("directory".equals(after.document().strategy().kind())) { + List regions = regions(before.document(), after.document()); + directoryExtents(before, regions).forEach((k, v) -> earlier.put(k, new DirExtent(v))); + directoryExtents(after, regions).forEach((k, v) -> later.put(k, new DirExtent(v))); + } else { + ringExtents(before.placement().shards()) + .forEach((k, v) -> earlier.put(k, new RingExtent(v))); + ringExtents(after.placement().shards()) + .forEach((k, v) -> later.put(k, new RingExtent(v))); + } + return List.of(earlier, later); + } + /** * For each shard the later snapshot names, the shards of the earlier one its extent draws from, * under {@code LIN-004}. @@ -154,18 +290,17 @@ private static boolean equal(List first, List second) { public static Map> parents(PlacementEngine before, PlacementEngine after) { List earlier = before.placement().shards(); List later = after.placement().shards(); - if (earlier.equals(later) || !"ring".equals(after.document().strategy().kind())) { + if (earlier.equals(later) || "slot".equals(after.document().strategy().kind())) { Map> identity = new LinkedHashMap<>(); later.forEach(shard -> identity.put(shard, List.of(shard))); return identity; } - Map> earlierExtents = ringExtents(earlier); - Map> laterExtents = ringExtents(later); + List> extents = geometry(before, after); Map> out = new LinkedHashMap<>(); for (String shard : later) { List drawn = new ArrayList<>(); for (String candidate : earlier) { - if (meets(earlierExtents.get(candidate), laterExtents.get(shard))) { + if (meetsExtent(extents.get(0).get(candidate), extents.get(1).get(shard))) { drawn.add(candidate); } } @@ -193,30 +328,26 @@ public static List classify(PlacementEngine before, PlacementEngine after } List earlier = before.placement().shards(); List later = after.placement().shards(); - if (!"ring".equals(kind)) { - if (!earlier.equals(later)) { - throw new Refused("unalignedLineage", - "the two snapshots enumerate different shards"); - } - return identity(later, before, after, replicaSet); - } - if (earlier.equals(later)) { + if ("slot".equals(kind) || earlier.equals(later)) { + // LIN-007 and LIN-012: an equal shard set is an equal extent set, and TOPO-231 has + // already refused a slot pair whose slotCount differs. return identity(later, before, after, replicaSet); } - Map> earlierExtents = ringExtents(earlier); - Map> laterExtents = ringExtents(later); + List> extents = geometry(before, after); + Map earlierExtents = extents.get(0); + Map laterExtents = extents.get(1); Set enumeratedLater = new LinkedHashSet<>(later); List entries = new ArrayList<>(); for (String shard : later) { - List mine = laterExtents.get(shard); + Extent mine = laterExtents.get(shard); List drawn = new ArrayList<>(); for (String candidate : earlier) { - List theirs = earlierExtents.get(candidate); - if (!meets(theirs, mine)) { + Extent theirs = earlierExtents.get(candidate); + if (!meetsExtent(theirs, mine)) { continue; } - if (!contains(theirs, mine) && !contains(mine, theirs)) { + if (!containsExtent(theirs, mine) && !containsExtent(mine, theirs)) { throw new Refused("unalignedLineage", shard); } drawn.add(candidate); @@ -225,7 +356,7 @@ public static List classify(PlacementEngine before, PlacementEngine after entries.add(new Entry(shard, Lineage.FRESH, List.of())); } else if (drawn.size() > 1) { entries.add(new Entry(shard, Lineage.MERGED, List.copyOf(drawn))); - } else if (equal(earlierExtents.get(drawn.get(0)), mine)) { + } else if (equalExtent(earlierExtents.get(drawn.get(0)), mine)) { entries.add(new Entry(shard, moved(shard, before, after, replicaSet), List.copyOf(drawn))); } else { @@ -236,10 +367,10 @@ public static List classify(PlacementEngine before, PlacementEngine after if (enumeratedLater.contains(shard)) { continue; } - List mine = earlierExtents.get(shard); + Extent mine = earlierExtents.get(shard); List children = new ArrayList<>(); for (String candidate : later) { - if (meets(laterExtents.get(candidate), mine)) { + if (meetsExtent(laterExtents.get(candidate), mine)) { children.add(candidate); } } From e58320eca516c9f9baaa8be5689cdf30f85039b1 Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 09:05:57 -0700 Subject: [PATCH 5/7] Expose the lineage and orient the user guide around it The user guide was reviewed against the code and three claims did not survive it. Two were mine and one predates this work. `LIN-031` requires the lineage to be an operation the integrator calls, and the port had none. `ShardExtents` sits in `core.internal.placement`, which `module-info.java` does not export, and `HandoffCoordinator` exposed `plan` alone. The conformance harness reached the classification by calling the internal class from inside the module, so the `migration` level passed over a requirement nothing implemented. `HandoffCoordinator` now carries `lineage(from, to)`, `ShardLineage` and `LineageClass` are exported beside it, and the harness goes through that surface, so a port that computes a lineage it cannot expose now fails the level rather than passing it. `30-conformance.md` no longer lists `LIN-031` among the requirements without an executable test, because it has one. The public `topology.OwnershipDelta` Javadoc still described the behaviour `TOPO-213` repaired. The internal copy was corrected with the repair and the exported one, which is the copy an integrator reads, was not. `lineage` refused a differing `topologyId` with the cause `topologyMismatch` while `plan` answers `incomparableShards` for the same input. Two operations disagreeing about one mistake, and the cause was the wrong one: its own documentation describes a plan named against a router that does not hold that topology, which is a different situation. Both now answer `incomparableShards`, and the interface documents all the refusals rather than three of four. The guide gains an orientation section before the first router, covering what a control plane holds and what the library does not do, how a published document reaches a running router and what installation replaces, and the everyday call from a key to the nodes that hold it. The last of those states where routing cost is paid rather than what it is: preparation is per snapshot and not per call, a call consumes the bounded prefix of `CORE-046` rather than the whole ordering, `rendezvous` is the exception because it scores the whole eligible set, and the whole-ordering calls belong off the routing path. It also gains the lineage operation beside the ownership delta, because the two are siblings that answer different questions: the delta joins on the shard identifier and answers who changed owner, the lineage joins on the keys a shard holds and answers where a new shard's contents are. The pair is the thing a reader needs and the guide did not carry it. One correction the review made to this author's own summary is worth recording: a shard that only the later snapshot enumerates is not absent from the ownership delta. It appears with an empty before set and every node in `gained`. What the delta cannot answer for it is where its contents are. The acceptance rule the guide stated was wrong and predates this work. It said the library accepts no document whose epoch is at or below the one in force. `TOPO-061` accepts an equal epoch with an equal digest as a no-op that refreshes freshness and refuses an equal epoch with a differing digest as a conflict rather than as stale, and `TopologyLoader` implements exactly that. Co-Authored-By: Claude Opus 5 --- conformance/coverage.json | 17 +- conformance/declarations/java.json | 6 +- conformance/generator/generate_extra.py | 6 +- conformance/manifest.json | 10 +- conformance/vectors/migration/lineage.json | 2 + docs/design/30-conformance.md | 9 +- ports/java/USER-GUIDE.md | 246 ++++++++++++++++-- ports/java/conformance/report.txt | 10 +- .../sharder/migrate/HandoffCoordinator.java | 17 +- .../sharder/migrate/LineageClass.java | 44 ++++ .../sharder/migrate/ShardLineage.java | 70 +++++ .../migrate/internal/DefaultCoordinator.java | 52 +++- .../sharder/topology/OwnershipDelta.java | 12 +- .../sharder/conformance/CoreVectors.java | 49 ++-- 14 files changed, 478 insertions(+), 72 deletions(-) create mode 100644 ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java create mode 100644 ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java diff --git a/conformance/coverage.json b/conformance/coverage.json index ae97bab..b0ff61a 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -1,8 +1,8 @@ { "specification": "docs/design/10-specification.md", "statedRequirements": 689, - "coveredRequirements": 473, - "uncoveredRequirements": 216, + "coveredRequirements": 474, + "uncoveredRequirements": 215, "coveragePercent": 68, "identifiersNamedButNotStated": [], "byPrefix": [ @@ -10,7 +10,7 @@ "prefix": "LIN", "section": "", "stated": 33, - "covered": 19, + "covered": 20, "uncovered": [ "LIN-001", "LIN-002", @@ -18,7 +18,6 @@ "LIN-005", "LIN-015", "LIN-023", - "LIN-031", "LIN-032", "LIN-034", "LIN-043", @@ -27,7 +26,7 @@ "LIN-055", "LIN-058" ], - "percent": 57 + "percent": 60 }, { "prefix": "CFG", @@ -1000,6 +999,7 @@ "LIN-016", "LIN-021", "LIN-022", + "LIN-031", "LIN-033", "LIN-041", "LIN-042", @@ -1081,8 +1081,8 @@ "TOPO-221", "TOPO-231" ], - "requirementsAtLevel": 102, - "requirementsWhenDeclared": 423 + "requirementsAtLevel": 103, + "requirementsWhenDeclared": 424 } ], "coveredBy": { @@ -1712,6 +1712,9 @@ "LIN-022": [ "vectors/migration/lineage.json" ], + "LIN-031": [ + "vectors/migration/lineage.json" + ], "LIN-033": [ "vectors/migration/lineage.json" ], diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index 4af016a..551e0d7 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49", + "revision": "62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5", "strategySurfaces": [ "directory", "rendezvous", @@ -42,8 +42,8 @@ "failures": 0 }, "scale": { - "wallTimeMs": 873, - "peakResidentBytes": 601296896, + "wallTimeMs": 865, + "peakResidentBytes": 597741568, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/generator/generate_extra.py b/conformance/generator/generate_extra.py index d0e7d25..f52bdbd 100644 --- a/conformance/generator/generate_extra.py +++ b/conformance/generator/generate_extra.py @@ -473,7 +473,7 @@ def path(name): cases = [] for label, before, after, requirements, note in [ ("ring-extent-divided", "delta-ring-before", "delta-ring-added", - ["LIN-004", "LIN-011", "LIN-021", "LIN-033"], + ["LIN-004", "LIN-011", "LIN-021", "LIN-031", "LIN-033"], "`LIN-011`: the added token divides `(1000, 3000]` into `(1000, 2000]` and " "`(2000, 3000]`, so both shards of the later snapshot are `divided` from one parent and " "the shard whose extent did not move is `moved` because its replica set did."), @@ -532,8 +532,8 @@ def path(name): "`directory` prefix refined and the fresh extent beside it, the incomparable pair, and " "the kind that enumerates no shard.", ["LIN-004", "LIN-006", "LIN-007", "LIN-011", "LIN-012", "LIN-013", "LIN-014", "LIN-016", - "LIN-021", "LIN-022", "LIN-033", "DIR-002", "DIR-010", "PLACE-065", "TOPO-231", - "MOVE-241", "ERR-050"], cases, level="migration") + "LIN-021", "LIN-022", "LIN-031", "LIN-033", "DIR-002", "DIR-010", "PLACE-065", + "TOPO-231", "MOVE-241", "ERR-050"], cases, level="migration") plans = [] for label, before, after, requirements, note in [ diff --git a/conformance/manifest.json b/conformance/manifest.json index db2be58..088004c 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,7 +1,7 @@ { "suite": "sharder conformance suite", "revision": { - "id": "e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49", + "id": "62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", "fileCount": 205, "trees": [ @@ -21,7 +21,7 @@ "scenarios": 25, "scenarioSteps": 313, "properties": 28, - "requirementsNamed": 473 + "requirementsNamed": 474 }, "requirementsNamed": [ "CFG-014", @@ -199,6 +199,7 @@ "LIN-016", "LIN-021", "LIN-022", + "LIN-031", "LIN-033", "LIN-041", "LIN-042", @@ -591,7 +592,7 @@ "vectorCases": 26, "scenarios": 15, "properties": 3, - "requirementsNamed": 102 + "requirementsNamed": 103 } ], "strategySurfaces": [ @@ -2080,12 +2081,13 @@ "LIN-016", "LIN-021", "LIN-022", + "LIN-031", "LIN-033", "MOVE-241", "PLACE-065", "TOPO-231" ], - "sha256": "1e13da445183131e15811f7b28046afd0094a7bc21692f6ecf2aff6e0007e9b2" + "sha256": "9a705c589dc85a46f81d0d3c9eee12883b170fe29ec8edb3088849a3ad273138" }, { "file": "vectors/migration/plan-construction.json", diff --git a/conformance/vectors/migration/lineage.json b/conformance/vectors/migration/lineage.json index 8f7e51e..4edcebc 100644 --- a/conformance/vectors/migration/lineage.json +++ b/conformance/vectors/migration/lineage.json @@ -17,6 +17,7 @@ "LIN-016", "LIN-021", "LIN-022", + "LIN-031", "LIN-033", "MOVE-241", "PLACE-065", @@ -29,6 +30,7 @@ "LIN-004", "LIN-011", "LIN-021", + "LIN-031", "LIN-033" ], "before": "topologies/delta-ring-before.topology.json", diff --git a/docs/design/30-conformance.md b/docs/design/30-conformance.md index 7c88047..c2ee5fa 100644 --- a/docs/design/30-conformance.md +++ b/docs/design/30-conformance.md @@ -763,9 +763,12 @@ forbids reading health, a clock, or a handoff state, and the second forbids the changing a class. An implementation that departs from either produces a different classification wherever the departure reaches one, and fails a lineage case there. -`LIN-031` and `LIN-032` are the shape of an operation and a rule about where it is not called, -which is the case `Surface exposure` and `Concurrency and visibility` describe. `LIN-034` bounds -what a lineage costs, which is the case `Placement cost` describes. +`LIN-032` is a rule about where an operation is not called, which is the case +`Concurrency and visibility` describes, and `LIN-034` bounds what a lineage costs, which is the case +`Placement cost` describes. `LIN-031` is not among them: it requires the lineage to be an operation +the integrator calls, so `vectors/migration/lineage.json` reaches it through the surface a port +exports rather than through the class behind it. A port that computed a lineage it could not expose +would satisfy the classification and fail the requirement that names it. `LIN-015` binds a registered strategy outside the core set. The suite carries core-set documents, so no data file can present one, and the same is true of every requirement about a registered strategy. diff --git a/ports/java/USER-GUIDE.md b/ports/java/USER-GUIDE.md index 7f53938..418b634 100644 --- a/ports/java/USER-GUIDE.md +++ b/ports/java/USER-GUIDE.md @@ -1,9 +1,9 @@ # Java port user guide -A task-oriented walk through the Java port of the sharder library: a first routed key, the node set -it routes over, replication, failover, read affinity, fencing, resharding, and orchestrated -migration. Each section assumes the ones above it, and each ends at the reference that carries the -full detail. +A task-oriented walk through the Java port of the sharder library: how a control plane and a caller +divide the work, a first routed key, the node set it routes over, replication, failover, read +affinity, fencing, resharding, and orchestrated migration. Each section assumes the ones above it, +and each ends at the reference that carries the full detail. ## Scope of the library @@ -29,6 +29,101 @@ function of the snapshot in force, the routing key, and the signals reported. A caller is the application embedding the library. It is not the client that talks to a node; that one belongs to the integrator's world. +## The control plane and the caller + +### Division of labour + +A topology authority decides what the cluster looks like and publishes a topology document for each +epoch. The authority sits outside the library: a control plane, a key-value store, a file under +configuration management, or an operator with an editor. The library assigns no epoch, elects no +authority, and writes no document. + +Each process that routes embeds the library and holds a `Router` over the document the authority +published. A router is long-lived. It is constructed once, it holds the snapshot in force, and it is +closed at shutdown; one constructed per request would repeat the preparation a snapshot pays for +once. + +| Lifetime | What is held | +|---|---| +| the process | one `Router`, and the `TopologyProvider` it reads documents from | +| an epoch | the immutable `TopologySnapshot` the router installed | +| one request | the `RoutingDecision`, the `AttemptSequence` over it, and its `FencingToken` | + +The authority publishes and the caller routes, and at run time neither asks anything of the other. +Two callers holding the same topology identifier and epoch compute the same preference list for the +same key, so no caller is told which node to use and none asks. +[`00-overview.md`](../../docs/design/00-overview.md#component-model) draws the components and the +three extension points an integrator implements. + +### A document reaching a running router + +A topology document is JSON, and where it comes from is the integrator's: a file on disk, a +key-value store, or octets a control plane already holds. It reaches a router through a +`TopologyProvider`, which delivers those octets and never constructs a snapshot. Validation, the +canonical form, the digest, the epoch comparison, and the preparation are the library's. +[Topology providers](#topology-providers) gives the interface, and the two providers the library +carries are the sections below it. + +An epoch is published and never inferred. The library assigns none and increments none. It compares +a candidate document against the snapshot in force by integer comparison of `epoch` and octet +comparison of `topologyId`, reading no clock, no provider revision, and no document timestamp. A +document below the epoch in force is refused as stale, one at that epoch carrying the same digest is +a no-op that refreshes freshness, one at that epoch carrying a different digest is refused as a +conflict, and one above it is installed. +[`10-specification.md`](../../docs/design/10-specification.md#monotonicity-and-acceptance) carries +the whole acceptance table. + +Installation replaces the snapshot in force whole, as a single atomic replacement of the reference a +routing call reads. A routing call already under way read that reference at entry and computes its +whole result from the snapshot it read, so an installation changes no call in flight and invalidates +no decision already answered. The router retains a bounded number of earlier snapshots for +[the recipient check](#the-recipient-check), and routes against none of them. + +### The nodes for one key + +The everyday call takes a key and answers the nodes that hold it, in the order to ask them. + +```java +RoutingDecision decision = router.route("tenant-42", RouteOptions.DEFAULTS); +NodeId owner = decision.primary().node(); +List candidates = decision.entries(); +FencingToken token = decision.token(); +``` + +`primary()` is the head of the list and the node to ask first. `entries()` holds that head and the +nodes behind it, each entry carrying its position, its `REPLICA` or `FALLBACK` role, its health +state, and whether the health filter left it attemptable. `shard()` names the shard the key belongs +to under the strategies that enumerate shards, and `token()` is what a request carries so the +recipient can check it. [The routing decision](#the-routing-decision) gives every member. + +Trying the nodes in turn is `router.attempts(decision)` rather than a loop over `entries()`, because +the walk applies the health filter, the attempt limit, and the retry budget. +[Walking an attempt sequence](#walking-an-attempt-sequence) gives it. + +The cost is bounded, and where it is paid matters more than what it is. + +- A routing call is a pure function of the snapshot in force, the routing key, and the health + signals the caller reported. It reads the snapshot reference once at entry and takes no lock. +- The work that grows with the node set is preparation, and preparation is performed once per + snapshot at installation rather than once per call. + [`10-specification.md`](../../docs/design/10-specification.md#placement-cost-model) bounds + preparation, one routing call, and the resident size of a prepared placement. +- A call consumes a bounded prefix of the candidate ordering rather than the whole ordering: the + greater of the achieved replica count and the resolved attempt limit, raised by the entries the + health filter skips. `CORE-046` fixes the prefix and `PLACE-071` bounds it, and + [`adr/0034`](../../docs/design/adr/0034-lazy-candidate-traversal-surface.md) records the cursor + that leaves the ordering beyond the prefix uncomputed. +- `rendezvous` is the exception. It determines its first candidate from the scores of the whole + eligible node set, so no prefix of its ordering costs less than the whole; `ring`, `slot`, and + `directory` pay for the prefix alone. +- `preferenceList()` and an explain record each consume the whole ordering, so both belong off the + routing path. + +The Java build holds an allocation gate over one routing call, in +`src/test/java/com/codeheadsystems/sharder/api/AllocationGateTest.java`. A ring of a hundred nodes +and a ring of a thousand take one ceiling between them, and a rendezvous topology takes a ceiling of +its own. + ## A first router ### Dependency coordinates @@ -600,17 +695,17 @@ contents and needs no handoff. ### A new epoch -An epoch is published, never inferred. The library assigns no epoch, increments none, and accepts no -document whose epoch is at or below the one in force under the same identifier. A change is a whole -document at the next epoch. +A change is a whole document at the next epoch, published the way every other document reaches a +router. ```java provider.publish(documentAtEpochTwo); ``` -The snapshot in force is replaced whole at installation. A routing call under way reads the snapshot -it started against from first candidate to last, so an installation changes no decision already -taken. +The epoch belongs to the authority, and what the library does with an arriving document is under +[A document reaching a running router](#a-document-reaching-a-running-router). A document below the +epoch in force is refused, so an epoch that carried a split is not undone by republishing the epoch +before it; the way back is a further epoch that folds the pieces again. ### The ownership delta @@ -638,15 +733,82 @@ The delta is computed on demand rather than at installation. Its entries come in the shards the later snapshot enumerates, in that snapshot's order, then the shards only the earlier one enumerates, so a shard that vanished reports the nodes that lost it rather than going unreported. Two snapshots whose shard identity is not comparable refuse the computation with -`InvalidArgumentException` rather than reporting every shard as wholly changed. +`InvalidArgumentException` rather than reporting every shard as wholly changed. The same pair given +to `plan` is refused with `PlanRefusedException` and the cause `incomparableShards`, so one +condition reaches a caller as whichever condition the surface it called raises. Under `rendezvous` the delta is empty, because the strategy enumerates no shards. Reading the delta is enough where the integrator's store moves data by itself. Where each move has -to be sequenced so that no request is served by both the old owner and the new one, the coordinator -is the next section. +to be sequenced so that no request is served by both the old owner and the new one, +[Orchestrated migration](#orchestrated-migration) gives the coordinator. [`10-specification.md`](../../docs/design/10-specification.md#ownership-delta) states the delta. +### The lineage operation + +`coordinator.lineage(from, to)` answers where each shard's contents come from across two snapshots. +It installs nothing and plans nothing, so an integrator sizes a change before deciding to run it, +which is the reason [the ownership delta](#the-ownership-delta) is a call rather than a product of +installation and the reason this is one too. + +The two answer different questions, and an epoch that changes which shards exist needs both. + +| | Ownership delta | Lineage | +|---|---|---| +| Joins on | the shard identifier | the keys a shard holds | +| Answers | which shards changed owner | where a shard's contents are | +| A shard that appeared | reports it gaining every replica | names the parents its extent draws from | +| A shard whose extent grew and kept its replicas | no entry | `MERGED`, with its parents | +| Under `rendezvous` | empty | refused | + +```java +ShardLineage lineage = coordinator.lineage(before, after); +for (ShardLineage.Entry entry : lineage.entries()) { + ShardId shard = entry.shard(); + LineageClass became = entry.lineage(); + List parents = entry.parents(); +} +``` + +`LineageClass` holds eight values. `UNCHANGED` and `MOVED` are the two whose extent is the one the +parent held, which `extentUnchanged()` reports, and which every shard falls into where an epoch +moved ownership alone. + +| Class | What became of the shard | +|---|---| +| `UNCHANGED` | one parent of equal extent, and the replica set is equal | +| `MOVED` | one parent of equal extent, and the replica set differs | +| `DIVIDED` | one parent whose extent strictly contains this one | +| `MERGED` | two or more parents whose extents this one contains | +| `FRESH` | no parent, this extent meeting no extent of the earlier snapshot | +| `SPLIT` | only the earlier snapshot enumerates it, and its extent divides into two or more | +| `FOLDED` | only the earlier snapshot enumerates it, and one child's extent contains it | +| `VACATED` | only the earlier snapshot enumerates it, and its extent meets no later extent | + +`parents()` names the shards of the earlier snapshot for an entry the later snapshot enumerates, and +the shards of the later one for an entry only the earlier snapshot enumerates. It is empty under +`FRESH` and `VACATED`. The entries come in the two groups the ownership delta comes in: first the +shards the later snapshot enumerates, in that snapshot's order, then the shards only the earlier one +enumerates. + +`lineage.entry(shard)` answers one entry. `lineage.counts()` answers how many shards fall in each +class, which is what the `migration.lineage` event reports. `lineage.identity()` answers whether +every shard kept the extent it held, which is every pair under `slot` and every pair under any kind +whose shard set did not move. + +The call refuses with `PlanRefusedException`, whose `reason()` carries one of three causes. Two +snapshots carrying different topology identifiers, and two whose shard identity is not comparable, +both give `incomparableShards`, because a pair under two identifiers joins on nothing and `plan` +answers the same cause for the same input. A strategy that enumerates no shard, which is +`rendezvous`, gives `strategyUnsupported`. A boundary that moved without either dividing or folding +an extent whole gives `unalignedLineage`, which is [the unaligned change](#the-unaligned-change). + +A snapshot this library did not produce is an `InvalidArgumentException` rather than a refusal, as +it is for `plan`. + +[`10-specification.md`](../../docs/design/10-specification.md#lineage-classification) states the +classes. + ## Orchestrated migration ### Store and strategy preconditions @@ -672,8 +834,26 @@ two snapshots and the policy, which is what lets a restarted coordinator rebuild plan is never created as a side effect of installing a snapshot. `plan` refuses with `PlanRefusedException` where the two snapshots do not join on shard identity, -where the target epoch does not advance, where the strategy supports no orchestrated migration, or -where a destination sits outside the target placement set. +where the target epoch does not advance, where the strategy supports no orchestrated migration, +where a destination sits outside the target placement set, where the change is unaligned, or where +the change needs a local step the hooks declare no support for. `PlanRefusedException.Cause` names +each of them. + +The handoffs come from the lineage rather than from the ownership delta. One handoff is admitted for +each destination of each shard of the later snapshot that does not already hold the contents of a +parent that shard's extent draws from, so a shard whose extent grew carries a handoff even where its +replica set is unchanged and the delta reports nothing for it. A shard with no parent has no +contents to move and carries none, and no handoff names one node as both its source and its +destination. +[`10-specification.md`](../../docs/design/10-specification.md#plan-construction-over-a-lineage) +states the derivation. + +A split or a merge also needs work on a node that holds the parent under the earlier snapshot and +the child under the later one, and that work moves nothing between nodes: the node divides or folds +its own copy so that what it holds matches the extent it owns. A plan carries it as a local step, +whose source and destination are the same node, sequenced before every handoff that draws from a +divided parent and after every handoff that draws into a folded shard. A local step counts against +the policy's concurrency as a handoff does. A plan is passive. It advances when `step` is called and never on a timer, a thread, or an installation. One `step` advances at most one handoff by at most one hook call. Concurrent calls to @@ -722,28 +902,45 @@ public interface MovementHooks { VerifyResult verify(HandoffContext context); HookResult cleanup(HandoffContext context); HookResult rollback(HandoffContext context); + HookResult divide(HandoffContext context); + HookResult combine(HandoffContext context); ObserveResult observe(HandoffContext context); } ``` +`divide` and `combine` are the hooks a local step calls, and each carries a default that refuses +permanently. A storage that has not implemented them declares `supportsLineage` of false, and the +plan that would call one is refused before any call is made. + Every hook is called from whichever unit of execution called `step`, at most one per call. A hook that raises rather than answering has its condition surfaced unchanged. -`HandoffContext` tells a hook the shard, the topology identifier, the source epoch and the target -epoch, the source node, the destination node, the attempt number, and the deadline in milliseconds. -The target epoch is the handoff's own rather than the plan's, so the two differ after a rebase. +`HandoffContext` tells a hook the shard, the shard its contents come from, the topology identifier, +the source epoch and the target epoch, the source node, the destination node, the attempt number, +and the deadline in milliseconds. `sourceShardId` differs from `shardId` only where a lineage +divided or folded an extent, and a hook that read `shardId` alone would search the source for a +shard the source does not hold; the convenience constructor defaults it to `shardId`. The target +epoch is the handoff's own rather than the plan's, so the two differ after a rebase. -`declare()` is read once per plan. +`declare()` is read once per plan. A `HookDeclaration` carries four components: the budget unit, and +whether the hooks support rollback, verification, and lineage. ```java HookDeclaration declaration = HookDeclaration.of("rows"); -HookDeclaration withoutVerification = new HookDeclaration("rows", true, false); +HookDeclaration dividing = HookDeclaration.withLineage("rows"); +HookDeclaration withoutVerification = new HookDeclaration("rows", true, false, false); ``` `budgetUnit` is opaque to the library: it is reported and never interpreted, and the unit counts a transfer and a catch-up answer are summed and compared and nothing else. `supportsVerify` of false is what lets `cleanup` follow a committed cutover directly, with no `verify` call. +`supportsLineage` states whether the integrator's storage can divide a copy in place and fold two +adjacent copies back together, which is a property of that storage rather than of the strategy. `of` +answers false for it and `withLineage` answers true. A plan that needs a local step over hooks that +declare false is refused with the cause `lineageUnsupported`, rather than reaching `DIVIDING` with +no hook to call and no way back. + `HookResult` is sealed over `Success`, `Deferred`, `Retryable`, and `Permanent`. `Retryable` is retried up to `maxAttemptsPerStep` with the policy's backoff, `Permanent` is never retried, and `Deferred` counts against no attempt budget and is not retried before its interval has elapsed on @@ -758,12 +955,13 @@ and `Permanent` are the two failures. ### The handoff states -`HandoffState` holds eleven values. `COMPLETE`, `ABORTED`, and `FAILED` are terminal, and nothing +`HandoffState` holds twelve values. `COMPLETE`, `ABORTED`, and `FAILED` are terminal, and nothing leaves a terminal state except a re-observation the integrator calls for one named handoff. | State | What is happening | |---|---| | `PLANNED` | admitted to the plan, no hook called | +| `DIVIDING` | a node holding both the parent and the child is dividing or folding its own copy | | `PREPARING` | the destination is being made ready to receive | | `TRANSFERRING` | the bulk contents are being copied | | `CATCHING_UP` | the residue accumulated during the copy is being closed | @@ -787,7 +985,11 @@ that horizon. Mutual exclusion across a cutover therefore rests on the single-wi assumption about the integrator's two clocks that the library states and cannot verify. `cleanup` never runs before the copy is established, and `rollback` never runs after a cutover -record belonging to the handoff exists. +record belonging to the handoff exists. A local step aborted in `DIVIDING` is undone by the inverse +hook, a division by `combine` and a fold by `divide`. Where the inverse fails and its attempts are +spent, the handoff reaches `FAILED` with the failure kind `undivided`, which names a copy on one +node matching neither the parent's extent nor the child's; each of the other four kinds names a +condition of a copy that moved between nodes. ### Rate control diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index dba7d73..9946d19 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,9 +1,9 @@ sharder conformance report, Java port -suite revision e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49 +suite revision 62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5 hash 2 of 2 vector files, 34 of 34 cases, 0.01s - place 45 of 45 vector files, 291 of 291 cases, 0.10s - core 8 of 8 vector files, 135 of 135 cases, 8.27s + place 45 of 45 vector files, 291 of 291 cases, 0.09s + core 8 of 8 vector files, 135 of 135 cases, 7.98s scale 2 of 2 vector files, 2 of 2 cases, 0.87s failover 3 of 3 vector files, 67 of 67 cases, 0.00s readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s @@ -12,6 +12,6 @@ suite revision e7b47c5f5ee68e66304d7b7524ac620f0b362a889208613f7cf4b001b87bcc49 609 cases, 0 failures scale wall time 0.87s - scale wall time 873 ms - peak resident size 601296896 bytes (573 MiB) + scale wall time 865 ms + peak resident size 597741568 bytes (570 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java index 3dfb8ef..b4292d9 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java @@ -9,7 +9,6 @@ * it is given, computes the ownership delta between them, and answers a plan the integrator drives. * A plan is never created as a side effect of installing a snapshot, under {@code MOVE-101}. */ -@FunctionalInterface public interface HandoffCoordinator { /** @@ -22,4 +21,20 @@ public interface HandoffCoordinator { */ MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHooks hooks, MigrationPolicy policy); + + /** + * Where each shard's contents come from across the two snapshots, under {@code LIN-031}. + * + *

An integrator sizes a migration before deciding to run it, which is the reason + * {@code TOPO-212} makes the ownership delta a call rather than a product of installation, and + * the reason this is one too. Computing a lineage installs nothing and plans nothing. + * + *

It refuses with {@code PlanRefusedException} where the two snapshots carry differing + * {@code topologyId}s or do not join on shard identity, both under the cause + * {@code incomparableShards} of {@code TOPO-231}; where the strategy enumerates no shard, + * under {@code strategyUnsupported}; and where a boundary moved without either dividing or + * folding an extent whole, under {@code unalignedLineage} of {@code LIN-022}. A snapshot this + * library did not produce is an {@code InvalidArgumentException}, as it is for {@code plan}. + */ + ShardLineage lineage(TopologySnapshot from, TopologySnapshot to); } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java new file mode 100644 index 0000000..4f073c7 --- /dev/null +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java @@ -0,0 +1,44 @@ +package com.codeheadsystems.sharder.migrate; + +/** + * What became of one shard between two snapshots, under {@code LIN-021}. + * + *

A shard the later snapshot enumerates is classified against the earlier one, and a shard only + * the earlier snapshot enumerates is classified against the later one, so every shard of a lineage + * carries exactly one of these. A pair whose extents neither divide nor fold whole has no class at + * all: it is refused under {@code LIN-022} rather than classified. + */ +public enum LineageClass { + /** One parent, whose extent is equal, and the replica set is equal. */ + UNCHANGED("unchanged"), + /** One parent, whose extent is equal, and the replica set differs. */ + MOVED("moved"), + /** One parent, whose extent strictly contains this one. */ + DIVIDED("divided"), + /** Two or more parents, whose extents this one contains. */ + MERGED("merged"), + /** No parent: this extent meets no extent of the earlier snapshot. */ + FRESH("fresh"), + /** Only the earlier snapshot enumerates it, and two or more children divide its extent. */ + SPLIT("split"), + /** Only the earlier snapshot enumerates it, and one child's extent contains its own. */ + FOLDED("folded"), + /** Only the earlier snapshot enumerates it, and its extent meets no later extent. */ + VACATED("vacated"); + + private final String spelling; + + LineageClass(String spelling) { + this.spelling = spelling; + } + + /** The spelling a vector, a scenario, and an event join on. */ + public String spelling() { + return spelling; + } + + /** Whether the shard's extent is the one its parent held. */ + public boolean extentUnchanged() { + return this == UNCHANGED || this == MOVED; + } +} diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java new file mode 100644 index 0000000..482b784 --- /dev/null +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java @@ -0,0 +1,70 @@ +package com.codeheadsystems.sharder.migrate; + +import com.codeheadsystems.sharder.ShardId; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Where each shard's contents come from across an epoch, under {@code LIN-031}. + * + *

An ownership delta joins two snapshots on the shard identifier and answers which shards + * changed owner. Where an epoch changes which shards exist, that join has no answer for the ones + * that appeared or vanished. A lineage is the second join, over the keys a shard holds rather than + * over its name, and it is what a plan reads to decide where a divided shard's contents are. + * + *

It is derived from the two topology documents. No member of a document records it, and an + * implementation never computes one by sampling routing keys, under {@code LIN-001}. + * + *

Where the two snapshots enumerate the same shards the lineage is the identity: every shard is + * its own parent and nothing is divided, merged, fresh, or vacated. That is every pair under + * {@code slot}, and every pair under any kind whose shard set did not move. + */ +public record ShardLineage(List entries) { + + /** + * One shard's class and the shards its extent draws from. + * + *

{@code parents} names the shards of the earlier snapshot for an entry the later snapshot + * enumerates, and the shards of the later snapshot for one only the earlier snapshot + * enumerates. It is empty for a shard that is {@code FRESH} or {@code VACATED}. + */ + public record Entry(ShardId shard, LineageClass lineage, List parents) { + + /** The lists are copied, so a caller holds an entry across an installation unchanged. */ + public Entry { + parents = List.copyOf(parents); + } + } + + /** The entries are copied, in the order {@code LIN-033} fixes. */ + public ShardLineage { + entries = List.copyOf(entries); + } + + /** The entry for one shard, where the lineage carries one. */ + public Optional entry(ShardId shard) { + return entries.stream().filter(entry -> entry.shard().equals(shard)).findFirst(); + } + + /** + * How many shards fall in each class, which is what {@code migration.lineage} reports. + * + *

An operator reads this to see whether an epoch refined the keyspace or redistributed it. + * The library takes no position on which it should have been, under + * {@code adr/0091}. + */ + public Map counts() { + Map out = new TreeMap<>(); + for (Entry entry : entries) { + out.merge(entry.lineage(), 1L, Long::sum); + } + return Map.copyOf(out); + } + + /** Whether every shard kept the extent it held, which is the identity of {@code LIN-007}. */ + public boolean identity() { + return entries.stream().allMatch(entry -> entry.lineage().extentUnchanged()); + } +} diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java index cafba3d..f2e1ccf 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java @@ -1,6 +1,7 @@ package com.codeheadsystems.sharder.migrate.internal; import com.codeheadsystems.sharder.NodeId; +import com.codeheadsystems.sharder.ShardId; import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot; @@ -9,8 +10,12 @@ import com.codeheadsystems.sharder.migrate.HandoffCoordinator; import com.codeheadsystems.sharder.migrate.MigrationPlan; import com.codeheadsystems.sharder.migrate.MigrationPolicy; +import com.codeheadsystems.sharder.migrate.LineageClass; import com.codeheadsystems.sharder.migrate.MovementHooks; +import com.codeheadsystems.sharder.migrate.ShardLineage; import com.codeheadsystems.sharder.topology.TopologySnapshot; +import java.util.ArrayList; +import java.util.List; /** * Where a plan is built, under {@code MOVE-061}. @@ -75,15 +80,50 @@ private static com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan p return com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.of( source.engine(), target.engine(), policy.quiesceLeaseMarginMillis()); } catch (ShardExtents.Refused refusal) { - throw new PlanRefusedException(switch (refusal.cause()) { - case "unalignedLineage" -> PlanRefusedException.Cause.UNALIGNED_LINEAGE; - case "lineageUnsupported" -> PlanRefusedException.Cause.LINEAGE_UNSUPPORTED; - case "strategyUnsupported" -> PlanRefusedException.Cause.STRATEGY_UNSUPPORTED; - default -> PlanRefusedException.Cause.INCOMPARABLE_SHARDS; - }); + throw new PlanRefusedException(causeOf(refusal)); } } + @Override + public ShardLineage lineage(TopologySnapshot from, TopologySnapshot to) { + DocumentSnapshot source = snapshot(from, "from"); + DocumentSnapshot target = snapshot(to, "to"); + // The same refusal `plan` gives for the same input. `TOPO-211` and `LIN-004` both join two + // snapshots of one `topologyId`, and a pair under two identifiers joins on nothing, which + // is the condition `TOPO-231` names rather than the plan-against-router mismatch. + if (!source.topologyId().equals(target.topologyId())) { + throw new PlanRefusedException(PlanRefusedException.Cause.INCOMPARABLE_SHARDS); + } + List entries = new ArrayList<>(); + for (ShardExtents.Entry entry : classify(source, target)) { + List parents = entry.parents().stream().map(ShardId::of).toList(); + entries.add(new ShardLineage.Entry(ShardId.of(entry.shard()), + LineageClass.valueOf(entry.lineage().name()), parents)); + } + return new ShardLineage(entries); + } + + /** The classification, with a lineage refusal reported as the condition {@code ERR-050} names. */ + private static List classify(DocumentSnapshot source, + DocumentSnapshot target) { + try { + return ShardExtents.classify(source.engine(), target.engine(), + OwnershipDelta::replicas); + } catch (ShardExtents.Refused refusal) { + throw new PlanRefusedException(causeOf(refusal)); + } + } + + /** The closed-set cause of {@code ERR-050} that a lineage refusal carries. */ + private static PlanRefusedException.Cause causeOf(ShardExtents.Refused refusal) { + return switch (refusal.cause()) { + case "unalignedLineage" -> PlanRefusedException.Cause.UNALIGNED_LINEAGE; + case "lineageUnsupported" -> PlanRefusedException.Cause.LINEAGE_UNSUPPORTED; + case "strategyUnsupported" -> PlanRefusedException.Cause.STRATEGY_UNSUPPORTED; + default -> PlanRefusedException.Cause.INCOMPARABLE_SHARDS; + }; + } + private static DocumentSnapshot snapshot(TopologySnapshot snapshot, String name) { if (snapshot instanceof DocumentSnapshot document) { return document; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java b/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java index 31713d3..5813ec0 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java @@ -11,9 +11,15 @@ * The shards whose replica sets differ between two snapshots, under {@code TOPO-211}. * *

The delta is computed on demand rather than at installation, and it reads the replica prefix - * of each shard rather than the whole preference list. A shard the later snapshot does not - * enumerate is absent from the delta, and a shard the earlier one did not enumerate has an empty - * before set. + * of each shard rather than the whole preference list. Its entries come in the two groups + * {@code TOPO-213} fixes: first the shards the later snapshot enumerates, in that snapshot's + * order, then the shards only the earlier one enumerates. A shard the earlier snapshot did not + * enumerate has an empty before set, and one the later snapshot does not enumerate has an empty + * after set. + * + *

A delta joins the two snapshots on the shard identifier, so it answers which shards changed + * owner and not where a new shard's contents are. Where an epoch changes which shards exist, that + * second question is answered by {@code HandoffCoordinator.lineage}. */ public record OwnershipDelta(List changes) { diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java index 3561c3a..15124a4 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java @@ -19,7 +19,13 @@ import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; +import com.codeheadsystems.sharder.ShardId; +import com.codeheadsystems.sharder.core.Sharder; import com.codeheadsystems.sharder.error.ErrorCode; +import com.codeheadsystems.sharder.error.PlanRefusedException; +import com.codeheadsystems.sharder.migrate.HandoffCoordinator; +import com.codeheadsystems.sharder.migrate.ShardLineage; +import com.codeheadsystems.sharder.topology.TopologySnapshot; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HexFormat; @@ -295,40 +301,53 @@ void ownershipDelta(JsonObject testCase) { /** {@code lineage}: the classification of two snapshots' extents, under {@code LIN-021}. */ void lineage(JsonObject testCase) { - PlacementEngine before = engine(testCase, "before"); - PlacementEngine after = engine(testCase, "after"); + // LIN-031 requires the lineage to be an operation the integrator calls, so the harness + // reaches it through the exported surface rather than through the class behind it. A port + // that computed a lineage it could not expose would pass the one and fail the other. + HandoffCoordinator coordinator = Sharder.coordinator(); + TopologySnapshot from = snapshotOf(testCase, "before"); + TopologySnapshot to = snapshotOf(testCase, "after"); JsonObject expect = testCase.object("expect"); - ShardExtents.ReplicaSet replicas = OwnershipDelta::replicas; if (expect.find("lineageComputed").isPresent()) { JsonObject condition = expect.object("condition"); ErrorCode code = ErrorCode.ofName(condition.text("name")); assertThat(code.code()).as("condition code").isEqualTo(condition.get("code").asInt()); - assertThatThrownBy(() -> ShardExtents.classify(before, after, replicas)) - .as("refusal").isInstanceOf(ShardExtents.Refused.class); + assertThatThrownBy(() -> coordinator.lineage(from, to)) + .as("refusal").isInstanceOf(PlanRefusedException.class); try { - ShardExtents.classify(before, after, replicas); - } catch (ShardExtents.Refused refusal) { - assertThat(refusal.cause()).as("cause").isEqualTo(condition.text("cause")); + coordinator.lineage(from, to); + } catch (PlanRefusedException refusal) { + assertThat(refusal.reason().spelling()).as("cause") + .isEqualTo(condition.text("cause")); assertThat(code.causes()).as("cause is in the closed set") .contains(condition.text("cause")); } return; } - List entries = ShardExtents.classify(before, after, replicas); + List entries = coordinator.lineage(from, to).entries(); List expected = expect.array("lineage").elements(); assertThat(entries).as("lineage").hasSize(expected.size()); for (int index = 0; index < expected.size(); index++) { JsonObject entry = expected.get(index).asObject(); - ShardExtents.Entry actual = entries.get(index); - assertThat(actual.shard()).as("lineage[%d].shard", index) + ShardLineage.Entry actual = entries.get(index); + assertThat(actual.shard().asText()).as("lineage[%d].shard", index) .isEqualTo(entry.text("shard")); - assertThat(actual.lineage().name().toLowerCase(java.util.Locale.ROOT)) - .as("lineage[%d].class", index).isEqualTo(entry.text("class")); - assertThat(actual.parents()).as("lineage[%d].parents", index) - .isEqualTo(entry.array("parents").texts()); + assertThat(actual.lineage().spelling()).as("lineage[%d].class", index) + .isEqualTo(entry.text("class")); + assertThat(actual.parents().stream().map(ShardId::asText).toList()) + .as("lineage[%d].parents", index).isEqualTo(entry.array("parents").texts()); } } + /** The exported snapshot type over a document a vector named, for a coordinator call. */ + private TopologySnapshot snapshotOf(JsonObject testCase, String member) { + JsonObject document = source.readObject(testCase.text(member)); + return new com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot( + new PlacementEngine(com.codeheadsystems.sharder.core.internal.document + .TopologyDocument.parse(document)), + Digests.of(JcsWriter.canonicalise(document)), java.util.OptionalLong.empty()); + } + /** {@code planConstruction}: how a plan derives a handoff from a lineage, {@code LIN-041}. */ void planConstruction(JsonObject testCase) { PlacementEngine before = engine(testCase, "before"); From f41657568976036289876106745bc51073d92a81 Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 09:13:56 -0700 Subject: [PATCH 6/7] Enforce the configured identifier and the epoch floor `TOPO-071` requires an implementation to reject any document whose epoch is below `minEpoch`, including the first after a restart, and the first row of `TOPO-061` refuses a document whose `topologyId` differs from the one in force or configured. The Java port accepted both settings through `RouterConfig.Builder`, validated them, reported them in `router.configuration()` and then consulted neither. `TopologyLoader` had no field for either and was constructed with the strategy map alone, so a process configured with a floor installed a document below it and a process configured with an identifier adopted the first document it was handed whatever cluster it named. Both rows precede the row that installs a first document, which is the whole point of them: they are what a process has instead of a snapshot in force when it comes back up. An operator who sets `minEpoch` to stop a restart serving an older topology got no protection and no error, which is the worst shape a missing check can take. The Python reference has implemented both since it was written, in the row order the table gives, so this is a port defect rather than an ambiguity. Nothing detected it because nothing exercised it: `coverage.json` recorded `TOPO-071` as covered by `properties/properties.json` alone, through a property that names the identifier, and that property's witness is `topology-rollback`, which runs with neither setting configured and has no step below a floor. The identifier was claimed as covered while no artefact could reach it. `topology-acceptance-floor` reaches it now. It configures both, then refuses a document below the floor with nothing in force, refuses a foreign document that is also below the floor as a conflict rather than as stale, which is what fixes the row order rather than merely the rows, and installs the first document at the floor. It fails against the unenforced port and passes against this one. Two smaller repairs came with it. A scenario whose first document is refused has nothing in force, so both drivers now compare `epochInForce` only where the expectation states one; they previously assumed every rejection had a snapshot behind it. The user guide documents both settings, which it could not honestly do while they did nothing. Co-Authored-By: Claude Opus 5 --- conformance/coverage.json | 6 +- conformance/declarations/java.json | 6 +- conformance/driver/python/run_suite.py | 17 ++- conformance/generator/generate_scenarios.py | 52 +++++++ conformance/manifest.json | 23 ++- conformance/scenarios/index.json | 18 ++- .../scenarios/topology-acceptance-floor.json | 133 ++++++++++++++++++ ports/java/USER-GUIDE.md | 18 ++- ports/java/conformance/report.txt | 8 +- .../internal/document/TopologyLoader.java | 41 +++++- .../core/internal/route/DefaultRouter.java | 7 +- .../sharder/conformance/ScenarioRunner.java | 12 +- 12 files changed, 316 insertions(+), 25 deletions(-) create mode 100644 conformance/scenarios/topology-acceptance-floor.json diff --git a/conformance/coverage.json b/conformance/coverage.json index b0ff61a..57b9b0a 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -1317,6 +1317,7 @@ ], "TOPO-061": [ "properties/properties.json", + "scenarios/topology-acceptance-floor.json", "scenarios/topology-rollback.json", "vectors/digest/canonical-form.json" ], @@ -2480,7 +2481,8 @@ "scenarios/topology-rollback.json" ], "TOPO-071": [ - "properties/properties.json" + "properties/properties.json", + "scenarios/topology-acceptance-floor.json" ], "TOPO-081": [ "properties/properties.json", @@ -2492,6 +2494,7 @@ ], "ERR-031": [ "properties/properties.json", + "scenarios/topology-acceptance-floor.json", "scenarios/topology-rollback.json" ], "ERR-034": [ @@ -2598,6 +2601,7 @@ "properties/properties.json" ], "ERR-032": [ + "scenarios/topology-acceptance-floor.json", "scenarios/topology-rollback.json" ], "ERR-040": [ diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index 551e0d7..d10bbbd 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5", + "revision": "c29e6a8f3facafa56414e46851ada1610db5e64c016714a8dba915e08875d57c", "strategySurfaces": [ "directory", "rendezvous", @@ -42,8 +42,8 @@ "failures": 0 }, "scale": { - "wallTimeMs": 865, - "peakResidentBytes": 597741568, + "wallTimeMs": 869, + "peakResidentBytes": 614211584, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/driver/python/run_suite.py b/conformance/driver/python/run_suite.py index 89e5045..b60796c 100644 --- a/conformance/driver/python/run_suite.py +++ b/conformance/driver/python/run_suite.py @@ -637,6 +637,7 @@ def run_scenarios(root, levels, known, verbose): scenario = json.loads((root / entry["file"]).read_text()) retained = {} in_force = None + loader_setup = scenario.get("setup", {}).get("loader", {}) for step in scenario["steps"]: action = step["action"] try: @@ -648,15 +649,25 @@ def run_scenarios(root, levels, known, verbose): compare(action + ".digest", jcs_digest(document), step["expect"]["digest"]) candidate = Snapshot(document) - outcome, condition = accept(candidate, in_force) + # `TOPO-061` and `TOPO-071`: a scenario states the identifier and the epoch + # floor its process was configured with, and both are checked before the row + # that installs a first document. + outcome, condition = accept( + candidate, in_force, + min_epoch=loader_setup.get("minEpoch"), + expected_topology_id=loader_setup.get("expectedTopologyId")) compare(action + ".outcome", outcome, step["expect"]["outcome"]) compare(action + ".condition", condition, step["expect"]["condition"]) if outcome == "installed": if in_force is not None: retained[in_force.epoch] = in_force in_force = candidate - compare(action + ".epochInForce", in_force.epoch, - step["expect"]["epochInForce"]) + # A scenario whose first document is refused has nothing in force, so the + # expectation states no epoch and there is none to compare. + if "epochInForce" in step["expect"]: + compare(action + ".epochInForce", + None if in_force is None else in_force.epoch, + step["expect"]["epochInForce"]) checked += 1 elif action == "recipientCheck": snapshot = None if step.get("noSnapshot") else in_force diff --git a/conformance/generator/generate_scenarios.py b/conformance/generator/generate_scenarios.py index 8ad010d..5f60c00 100644 --- a/conformance/generator/generate_scenarios.py +++ b/conformance/generator/generate_scenarios.py @@ -37,6 +37,7 @@ # absent from this table. LEVEL_OF_SCENARIO = { "topology-rollback": "core", + "topology-acceptance-floor": "core", "caller-three-epochs-stale": "fencing", "split-topology-view": "fencing", "redirect-walk-depth-limit": "fencing", @@ -281,6 +282,56 @@ def build_rollback_scenario(): ["TOPO-051", "TOPO-061", "TOPO-081", "TOPO-091", "ERR-031", "ERR-032"], steps) +def build_acceptance_floor_scenario(): + """`TOPO-071` and the first row of `TOPO-061`: the floor and the configured identifier. + + Both are checked before the row that installs a first document, so both hold with nothing in + force, which is the state a process is in after a restart. The rest of the acceptance table + is covered by `topology-rollback`, which runs with neither configured. + """ + from sharder_ref.topology import accept as accept_document + + setup = {"loader": {"expectedTopologyId": "migration", "minEpoch": 3}} + steps = [] + for name, document, note in [ + ("migration-epoch-2", EPOCH2, + "`TOPO-071`: the floor is below the first document, and nothing is in force, so the " + "row that installs a first document is never reached"), + ("migration-epoch-3", EPOCH3, None), + ]: + outcome, condition = accept_document(Snapshot(document), None, min_epoch=3, + expected_topology_id="migration") + step = {"action": "installTopology", + "topology": "topologies/%s.topology.json" % name, + "expect": {"outcome": outcome, "condition": condition, + "digest": jcs_digest(document)}} + if note: + step["note"] = note + if outcome == "installed": + step["expect"]["epochInForce"] = document["epoch"] + steps.insert(len(steps) if name != "migration-epoch-3" else len(steps), step) + + # The identifier row precedes the floor row, so a foreign document below the floor is a + # conflict rather than stale. Ordering is the whole point of this step. + foreign = copy.deepcopy(EPOCH1) + foreign["topologyId"] = "some-other-cluster" + outcome, condition = accept_document(Snapshot(foreign), None, min_epoch=3, + expected_topology_id="migration") + steps.insert(1, { + "action": "installTopology", "document": foreign, + "note": "`TOPO-061`: the identifier row precedes the floor row, so a foreign document " + "below `minEpoch` is a conflict and not stale", + "expect": {"outcome": outcome, "condition": condition, "digest": jcs_digest(foreign)}, + }) + + register("topology-acceptance-floor", + "A process configured with an identifier and an epoch floor, with nothing in force. " + "The floor refuses a document below it, the configured identifier refuses a foreign " + "one before the floor is reached, and the first document at or above the floor " + "installs.", + ["TOPO-061", "TOPO-071", "ERR-031", "ERR-032"], steps, setup=setup) + + def build_stale_caller_scenario(): """A caller three epochs behind, at a recipient holding the newest snapshot.""" in_force = SNAP["migration-epoch-4"] @@ -1825,6 +1876,7 @@ def main(): build_redirect_scenario() build_happy_path_scenario() build_local_step_scenarios() + build_acceptance_floor_scenario() build_quiesce_lease_scenario() build_node_dies_scenario() build_abort_during_catchup_scenario() diff --git a/conformance/manifest.json b/conformance/manifest.json index 088004c..84f38bd 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,9 +1,9 @@ { "suite": "sharder conformance suite", "revision": { - "id": "62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5", + "id": "c29e6a8f3facafa56414e46851ada1610db5e64c016714a8dba915e08875d57c", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", - "fileCount": 205, + "fileCount": 206, "trees": [ "vectors", "topologies", @@ -18,8 +18,8 @@ "vectorFiles": 69, "vectorCases": 609, "topologyDocuments": 109, - "scenarios": 25, - "scenarioSteps": 313, + "scenarios": 26, + "scenarioSteps": 316, "properties": 28, "requirementsNamed": 474 }, @@ -530,7 +530,7 @@ "surface": "the document pipeline, the snapshot lifecycle, and what is reported", "vectorFiles": 8, "vectorCases": 135, - "scenarios": 1, + "scenarios": 2, "properties": 22, "requirementsNamed": 126 }, @@ -3321,6 +3321,19 @@ "MOVE-421" ] }, + { + "file": "scenarios/topology-acceptance-floor.json", + "scenario": "topology-acceptance-floor", + "level": "core", + "description": "A process configured with an identifier and an epoch floor, with nothing in force. The floor refuses a document below it, the configured identifier refuses a foreign one before the floor is reached, and the first document at or above the floor installs.", + "stepCount": 3, + "requirements": [ + "ERR-031", + "ERR-032", + "TOPO-061", + "TOPO-071" + ] + }, { "file": "scenarios/quiesce-lease-expiry.json", "scenario": "quiesce-lease-expiry", diff --git a/conformance/scenarios/index.json b/conformance/scenarios/index.json index f5f8603..e9c2dd5 100644 --- a/conformance/scenarios/index.json +++ b/conformance/scenarios/index.json @@ -1,7 +1,7 @@ { "suite": "sharder simulation scenarios", - "scenarioCount": 25, - "stepCount": 313, + "scenarioCount": 26, + "stepCount": 316, "requirementsCovered": [ "CFG-030", "CFG-040", @@ -142,6 +142,7 @@ "SEC-031", "TOPO-051", "TOPO-061", + "TOPO-071", "TOPO-081", "TOPO-091", "TOPO-111", @@ -305,6 +306,19 @@ "MOVE-421" ] }, + { + "file": "scenarios/topology-acceptance-floor.json", + "scenario": "topology-acceptance-floor", + "level": "core", + "description": "A process configured with an identifier and an epoch floor, with nothing in force. The floor refuses a document below it, the configured identifier refuses a foreign one before the floor is reached, and the first document at or above the floor installs.", + "stepCount": 3, + "requirements": [ + "ERR-031", + "ERR-032", + "TOPO-061", + "TOPO-071" + ] + }, { "file": "scenarios/quiesce-lease-expiry.json", "scenario": "quiesce-lease-expiry", diff --git a/conformance/scenarios/topology-acceptance-floor.json b/conformance/scenarios/topology-acceptance-floor.json new file mode 100644 index 0000000..2119521 --- /dev/null +++ b/conformance/scenarios/topology-acceptance-floor.json @@ -0,0 +1,133 @@ +{ + "scenario": "topology-acceptance-floor", + "level": "core", + "description": "A process configured with an identifier and an epoch floor, with nothing in force. The floor refuses a document below it, the configured identifier refuses a foreign one before the floor is reached, and the first document at or above the floor installs.", + "requirements": [ + "ERR-031", + "ERR-032", + "TOPO-061", + "TOPO-071" + ], + "deterministic": true, + "setup": { + "loader": { + "expectedTopologyId": "migration", + "minEpoch": 3 + } + }, + "steps": [ + { + "action": "installTopology", + "topology": "topologies/migration-epoch-2.topology.json", + "expect": { + "outcome": "rejected", + "condition": { + "code": 203, + "name": "staleDocument", + "detail": "an epoch below minEpoch" + }, + "digest": "88174beca86efc281ac17c76a28b20728055b2a526d2cd9a216a3ca5f2307bb8" + }, + "note": "`TOPO-071`: the floor is below the first document, and nothing is in force, so the row that installs a first document is never reached" + }, + { + "action": "installTopology", + "document": { + "formatVersion": "1.0", + "topologyId": "some-other-cluster", + "epoch": 1, + "domainLevels": [ + "zone" + ], + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "slot", + "slotCount": 3, + "assignment": "explicit", + "assignments": [ + { + "slots": [ + "0" + ], + "nodes": [ + "n1", + "n2" + ] + }, + { + "slots": [ + "1" + ], + "nodes": [ + "n2", + "n3" + ] + }, + { + "slots": [ + "2" + ], + "nodes": [ + "n3", + "n1" + ] + } + ] + }, + "nodes": [ + { + "id": "n1", + "domains": { + "zone": "za" + }, + "address": "10.0.0.1:7000" + }, + { + "id": "n2", + "domains": { + "zone": "zb" + }, + "address": "10.0.0.2:7000" + }, + { + "id": "n3", + "domains": { + "zone": "zc" + }, + "address": "10.0.0.3:7000" + }, + { + "id": "n4", + "state": "joining", + "domains": { + "zone": "zd" + }, + "address": "10.0.0.4:7000" + } + ] + }, + "note": "`TOPO-061`: the identifier row precedes the floor row, so a foreign document below `minEpoch` is a conflict and not stale", + "expect": { + "outcome": "rejected", + "condition": { + "code": 202, + "name": "topologyConflict", + "detail": "a differing topologyId" + }, + "digest": "5dc597d8f3c39ade58e5f7fcb16804f6ce3351d8da5859c295491b439f40343f" + } + }, + { + "action": "installTopology", + "topology": "topologies/migration-epoch-3.topology.json", + "expect": { + "outcome": "installed", + "condition": null, + "digest": "135af637f87fa8f69f32fe0be4f150501c505d8ce4c49db464c4c15856baa99c", + "epochInForce": 3 + } + } + ] +} diff --git a/ports/java/USER-GUIDE.md b/ports/java/USER-GUIDE.md index 418b634..2ce407b 100644 --- a/ports/java/USER-GUIDE.md +++ b/ports/java/USER-GUIDE.md @@ -70,8 +70,24 @@ comparison of `topologyId`, reading no clock, no provider revision, and no docum document below the epoch in force is refused as stale, one at that epoch carrying the same digest is a no-op that refreshes freshness, one at that epoch carrying a different digest is refused as a conflict, and one above it is installed. +Two settings are checked before any of that, and both hold when nothing is in force, which is the +state a process is in after a restart. `expectedTopologyId` names the cluster this process routes +for, so a document under any other identifier is refused as a conflict rather than adopted, which is +what an unconfigured router does with the first document it accepts. `minEpoch` is a floor, and a +document below it is refused as stale even where it is the first to arrive, which is how an operator +stops a process coming back up on a document older than the one it was serving. + +```java +RouterConfig config = RouterConfig.builder() + .provider(provider) + .expectedTopologyId("orders") + .minEpoch(41) + .build(); +``` + [`10-specification.md`](../../docs/design/10-specification.md#monotonicity-and-acceptance) carries -the whole acceptance table. +the whole acceptance table, including the order the rows are evaluated in: the identifier is checked +before the floor, so a foreign document below the floor is a conflict and not stale. Installation replaces the snapshot in force whole, as a single atomic replacement of the reference a routing call reads. A routing call already under way read that reference at entry and computes its diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index 9946d19..6ace4a9 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,9 +1,9 @@ sharder conformance report, Java port -suite revision 62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5 +suite revision c29e6a8f3facafa56414e46851ada1610db5e64c016714a8dba915e08875d57c hash 2 of 2 vector files, 34 of 34 cases, 0.01s place 45 of 45 vector files, 291 of 291 cases, 0.09s - core 8 of 8 vector files, 135 of 135 cases, 7.98s + core 8 of 8 vector files, 135 of 135 cases, 7.92s scale 2 of 2 vector files, 2 of 2 cases, 0.87s failover 3 of 3 vector files, 67 of 67 cases, 0.00s readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s @@ -12,6 +12,6 @@ suite revision 62fee56e23bddee56e5b186d465b8b4e6f936d851785af47ba0e30f8ce178db5 609 cases, 0 failures scale wall time 0.87s - scale wall time 865 ms - peak resident size 597741568 bytes (570 MiB) + scale wall time 869 ms + peak resident size 614211584 bytes (585 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java index 37dc5f2..8527a78 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java @@ -55,6 +55,8 @@ public record Arrival(Outcome outcome, Optional condition, Optional strategies; private final Map retained = new LinkedHashMap<>(); + private final Optional configuredTopologyId; + private final OptionalLong minEpoch; private String topologyId; private TopologyDocument inForce; private Digest digestInForce; @@ -68,7 +70,23 @@ public TopologyLoader() { /** A pipeline under the strategies a configuration registered, under {@code CORE-010}. */ public TopologyLoader(Map strategies) { + this(strategies, Optional.empty(), OptionalLong.empty()); + } + + /** + * The same, under the identifier and the epoch floor the integrator configured. + * + *

{@code TOPO-061} checks a configured identifier before there is a snapshot in force, so a + * first document under a foreign identifier is a conflict rather than an installation, and + * {@code TOPO-071} applies {@code minEpoch} to every document including the first after a + * restart. Both are refused before the row that installs a first document, which is why they + * belong to the pipeline rather than to a caller checking afterwards. + */ + public TopologyLoader(Map strategies, + Optional configuredTopologyId, OptionalLong minEpoch) { this.strategies = Map.copyOf(strategies); + this.configuredTopologyId = configuredTopologyId; + this.minEpoch = minEpoch; } /** The snapshot in force, where one is. */ @@ -86,9 +104,16 @@ public OptionalLong epochInForce() { return epochInForce; } - /** The identifier every later document is checked against, adopted from the first accepted. */ + /** + * The identifier every document is checked against. + * + *

It is the one the integrator configured where there is one, and otherwise the one adopted + * from the first accepted document under {@code TOPO-091}. + */ public Optional expectedTopologyId() { - return Optional.ofNullable(topologyId); + return configuredTopologyId.isPresent() + ? configuredTopologyId + : Optional.ofNullable(topologyId); } /** The document's arrival, which installs it, treats it as a no-op, or refuses it. */ @@ -101,10 +126,18 @@ public Arrival accept(JsonObject document) { } TopologyDocument parsed = TopologyDocument.parse(document); - // TOPO-061, in the order the table writes the rows. - if (topologyId != null && !topologyId.equals(parsed.topologyId())) { + // TOPO-061, in the order the table writes the rows. The first two precede the row that + // installs a first document, so a first document under a foreign identifier is a conflict + // and a first document below minEpoch is stale, rather than an installation. + Optional expected = expectedTopologyId(); + if (expected.isPresent() && !expected.get().equals(parsed.topologyId())) { return refused(ErrorCode.TOPOLOGY_CONFLICT, "a differing topologyId", digest); } + if (minEpoch.isPresent() && parsed.epoch() < minEpoch.getAsLong()) { + // TOPO-071: the floor holds across a restart, so it is checked whether or not a + // snapshot is in force. + return refused(ErrorCode.STALE_DOCUMENT, "an epoch below minEpoch", digest); + } if (inForce == null) { return install(parsed, digest); } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java index 803aad0..52ffccd 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java @@ -101,7 +101,12 @@ public final class DefaultRouter implements Router { public DefaultRouter(RouterConfig config) { this.config = config; this.provider = config.provider().provider(); - this.pipeline = new TopologyLoader(config.strategies()); + // TOPO-061 and TOPO-071: the configured identifier and the epoch floor belong to the + // pipeline that installs, because both are refused before the row that installs a first + // document. The standalone loader of `loader()` checks one document in isolation and + // applies neither. + this.pipeline = new TopologyLoader(config.strategies(), + config.provider().expectedTopologyId(), config.provider().minEpoch()); this.health = config.healthView().orElseGet( () -> new SlidingWindowHealthView(config.health(), config.hintObserver().orElse(null))); diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java index 567f441..023cfc0 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java @@ -42,7 +42,17 @@ final class ScenarioRunner { /** One scenario's steps, replayed in order, answering the count this driver skipped. */ int run(String path) { JsonObject scenario = source.readObject(path); - TopologyLoader loader = new TopologyLoader(); + // TOPO-061 and TOPO-071: a scenario states the identifier and the epoch floor its process + // was configured with, both of which the pipeline checks before it installs anything. + JsonObject loaderSetup = scenario.object("setup").find("loader") + .map(JsonValue::asObject).orElse(null); + TopologyLoader loader = loaderSetup == null + ? new TopologyLoader() + : new TopologyLoader(java.util.Map.of(), + loaderSetup.find("expectedTopologyId").map(JsonValue::asText), + loaderSetup.find("minEpoch") + .map(value -> java.util.OptionalLong.of(value.asLong())) + .orElseGet(java.util.OptionalLong::empty)); SlidingWindowHealthView[] health = {new SlidingWindowHealthView(HealthSettings.defaults())}; RetryBudget budget = RetryBudget.defaults(); int skipped = 0; From bea4494a308414b3910b7b067ce40ba9a4e4ef71 Mon Sep 17 00:00:00 2001 From: Ned Wolpert Date: Sat, 19 Sep 2026 09:21:42 -0700 Subject: [PATCH 7/7] Report the events of the migration surface `OBS-020` tables ten events under `migration.` and requires an implementation to emit the events of the surfaces it exposes. The Java port declared all ten in the inventory the `migration` vector reads, and emitted none of them. The cause was structural rather than ten missed call sites. A router is built from a `RouterConfig`, which carries the metrics registry and the event sink, so `DefaultRouter` holds a `MetricsHolder` and reports through it. A coordinator is built by `Sharder.coordinator()`, which takes nothing, so there was no sink to report through and nowhere for one to come from. 0092 settles where it comes from: `Sharder.coordinator(config)`, following `Sharder.loader(config)`, which already takes the whole configuration to answer a narrower question. The sinkless factory stays and means a coordinator that reports nothing. All ten now emit at the trigger `OBS-020` names, with the payload members it names. `state_changed` reports from the one chokepoint every transition passes through, and reports nothing for a trigger the machine dropped because the state moved under a hook, since no transition happened. `failed` follows it where the transition reached `failed`, carrying the kind of `MOVE-011`. `quiesce_expired` is raised only where the machine says the lease is spent, because an idle outcome in that position is otherwise a concurrent abort. Nothing in the suite checks any of this, and that is the part worth recording. No driver asserts that a running library emitted a named event, for any surface. `30-conformance.md` claimed otherwise: it said emission is asserted where the suite drives one, naming `sharder.health.ejection_refused` in the health scenarios. Those scenarios do carry an `ejectionsRefused` member, and neither driver reads it, so it is data rather than coverage. A port that declares an inventory it never emits from passes every level, which is exactly the state this port was in for the whole `migration` surface. The section now says that plainly, and a port's own tests are named as where emission is checked until a driver asserts it. `MigrationTest` carries that test for this port: it drives a handoff to completion through a coordinator with a sink and asserts the events, their severities, the common members of `OBS-021`, and the payload members of each. Co-Authored-By: Claude Opus 5 --- docs/README.md | 2 +- docs/design/30-conformance.md | 14 ++- ...-observability-sink-for-the-coordinator.md | 70 ++++++++++++ ports/java/USER-GUIDE.md | 5 + .../codeheadsystems/sharder/core/Sharder.java | 16 +++ .../migrate/internal/DefaultCoordinator.java | 39 ++++++- .../internal/DefaultMigrationPlan.java | 101 +++++++++++++++++- .../sharder/api/MigrationTest.java | 56 ++++++++++ 8 files changed, 290 insertions(+), 13 deletions(-) create mode 100644 docs/design/adr/0092-an-observability-sink-for-the-coordinator.md diff --git a/docs/README.md b/docs/README.md index bd19e87..e012b1d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -197,7 +197,7 @@ other port carries too. | placement and weights | [0002](design/adr/0002-placement-strategy-set.md), [0003](design/adr/0003-integer-node-weights.md), [0010](design/adr/0010-shard-identifier-naming.md), [0011](design/adr/0011-derived-assignment-virtual-nodes.md), [0013](design/adr/0013-range-bounds-over-routing-key.md), [0039](design/adr/0039-placement-cost-model-and-warning-thresholds.md), [0043](design/adr/0043-assignment-mode-defaults.md), [0054](design/adr/0054-range-strategy-withdrawal.md), [0055](design/adr/0055-slot-derived-assignment-withdrawal.md), [0071](design/adr/0071-candidate-ordering-over-a-placement-ring.md), [0072](design/adr/0072-preparation-cost-reported-before-preparation.md) | | topology model and format | [0004](design/adr/0004-topology-provider-contract.md), [0005](design/adr/0005-epoch-and-version-semantics.md), [0006](design/adr/0006-failure-domain-model.md), [0008](design/adr/0008-json-canonical-serialisation.md), [0009](design/adr/0009-override-composition.md), [0038](design/adr/0038-provider-contract-in-the-specification.md), [0048](design/adr/0048-conditional-fetch-in-the-provider-contract.md) | | replication and failover | [0007](design/adr/0007-administrative-state-and-health-state.md), [0012](design/adr/0012-balance-bound-tolerances.md), [0014](design/adr/0014-read-affinity-as-a-separate-call.md), [0015](design/adr/0015-spread-degradation-algorithm.md), [0016](design/adr/0016-node-health-state-machine.md), [0017](design/adr/0017-failover-depth-and-substitution.md), [0036](design/adr/0036-spread-relaxation-ladder-direction.md), [0042](design/adr/0042-domain-path-scope.md), [0045](design/adr/0045-attempt-limit-resolution-order.md), [0046](design/adr/0046-bounded-routing-decision-surface.md), [0063](design/adr/0063-decision-api-surface-boundaries.md), [0069](design/adr/0069-per-level-occupancy-cap.md), [0070](design/adr/0070-spread-stage-feasibility-from-a-domain-count.md), [0075](design/adr/0075-affinity-path-length-refusal.md) | -| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md), [0086](design/adr/0086-shard-lineage-derived-from-extent.md), [0087](design/adr/0087-lineage-capability-derived-rather-than-declared.md), [0089](design/adr/0089-a-division-undone-by-a-combination.md), [0090](design/adr/0090-no-rebase-across-a-lineage-boundary.md), [0091](design/adr/0091-lineage-classification-as-a-signal.md) | +| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md), [0086](design/adr/0086-shard-lineage-derived-from-extent.md), [0087](design/adr/0087-lineage-capability-derived-rather-than-declared.md), [0089](design/adr/0089-a-division-undone-by-a-combination.md), [0090](design/adr/0090-no-rebase-across-a-lineage-boundary.md), [0091](design/adr/0091-lineage-classification-as-a-signal.md), [0092](design/adr/0092-an-observability-sink-for-the-coordinator.md) | | cross-cutting contracts | [0024](design/adr/0024-closed-numbered-error-taxonomy.md), [0025](design/adr/0025-observability-contract-and-explain-record.md), [0026](design/adr/0026-configuration-defaults-and-locality.md), [0027](design/adr/0027-hash-seed-exposure-and-tenancy.md), [0060](design/adr/0060-node-label-cardinality-ceiling.md), [0076](design/adr/0076-ordered-rows-in-a-precedence-table.md) | | Java binding | [0028](design/adr/0028-java-module-and-artifact-layout.md), [0029](design/adr/0029-exception-idiom-for-the-taxonomy.md), [0031](design/adr/0031-jdk-baseline.md), [0032](design/adr/0032-dependency-free-json-and-canonicalisation.md), [0033](design/adr/0033-opaque-identifier-value-types.md), [0034](design/adr/0034-lazy-candidate-traversal-surface.md), [0041](design/adr/0041-exact-product-comparison-surface.md), [0081](design/adr/0081-single-java-module.md), [0085](design/adr/0085-hook-declarations-and-refused-aborts.md) | | conformance | [0035](design/adr/0035-manifest-driven-conformance-harness.md), [0037](design/adr/0037-specification-defect-repairs.md), [0052](design/adr/0052-conformance-level-partition.md), [0058](design/adr/0058-conformance-surfaces.md), [0059](design/adr/0059-place-conformance-level.md), [0061](design/adr/0061-suite-revision-identifier.md), [0064](design/adr/0064-hash-verification-before-generation.md), [0065](design/adr/0065-level-coverage-inside-surface-boundaries.md), [0077](design/adr/0077-scale-conformance-level.md), [0078](design/adr/0078-observability-contract-as-data.md), [0084](design/adr/0084-missing-member-as-a-validation-error.md), [0088](design/adr/0088-lineage-inside-the-migration-surface.md) | diff --git a/docs/design/30-conformance.md b/docs/design/30-conformance.md index c2ee5fa..dabaf64 100644 --- a/docs/design/30-conformance.md +++ b/docs/design/30-conformance.md @@ -493,10 +493,16 @@ assertion that a port warning about an ordinary topology fails. A port compares the inventory against its own registry and its own sink. A driver holds neither, so [`../../conformance/driver/python/run_suite.py`](../../conformance/driver/python/run_suite.py) checks the inventory's internal consistency and its surface assignment, as it does for the closed -condition set of `errorTaxonomy`. The events a running library emits are asserted where the suite -already drives one: the health scenarios carry `sharder.health.ejection_refused` in their -expectations. [`adr/0078`](adr/0078-observability-contract-as-data.md) records what is asserted and -what is not. +condition set of `errorTaxonomy`. [`adr/0078`](adr/0078-observability-contract-as-data.md) records +what is asserted and what is not. + +What the suite does not assert is emission. No driver checks that a running library emitted a named +event at the moment a requirement says it does, for any surface. A scenario carries the shape of +such an expectation in one place, the `ejectionsRefused` member the health scenarios hold, and no +driver reads that member either, so it is data rather than coverage. A port that declares an +inventory it never emits from passes every level, which is the state the Java port was in for the +whole `migration` surface. Until a driver asserts emission, a port's own tests are where emission is +checked, and the inventory is the contract the suite holds it to. ## Properties diff --git a/docs/design/adr/0092-an-observability-sink-for-the-coordinator.md b/docs/design/adr/0092-an-observability-sink-for-the-coordinator.md new file mode 100644 index 0000000..1776806 --- /dev/null +++ b/docs/design/adr/0092-an-observability-sink-for-the-coordinator.md @@ -0,0 +1,70 @@ +# 0092. An observability sink for the coordinator + +Date: 2026-09-19 + +Status: accepted + +## Context + +`OBS-020` tables ten events under `migration.`, and an implementation emits the events of the +surfaces it exposes. The Java port declared all ten in its inventory, which is what the +`migration` inventory vector reads, and emitted none of them. Nothing detected that, because no +driver asserts an emitted event for any surface. + +The reason the port emitted none is structural rather than an oversight at a call site. A router is +built from a `RouterConfig`, which carries the metrics registry and the event sink, so +`DefaultRouter` holds a `MetricsHolder` and emits the `routing` events through it. A coordinator is +built by `Sharder.coordinator()`, which takes nothing, so there is no sink to emit through and +nowhere for one to come from. + +The coordinator is deliberately stateless between calls under `MOVE-061`, and a plan is a pure +function of the two snapshots and the policy under `MOVE-221`. Neither of those is in tension with +reporting what happened, but both rule out the coordinator acquiring a sink by holding onto a +router. + +## Decision + +`Sharder.coordinator(RouterConfig)` is added beside `Sharder.coordinator()`. The configuration +carries the metrics registry and the event sink the router already reads, and the coordinator reads +the same two and nothing else from it. + +`Sharder.coordinator()` stays, and means a coordinator that reports nothing. It is the right call +for an integrator that has not configured observability, and removing it would make the simple case +carry a configuration it has no other use for. + +`MigrationPolicy` does not carry the sink. It is a record of tuning values that a plan reports as a +payload member of `migration.planned`, and putting a dependency inside it would make two plans +incomparable that differ only in where their events go. + +`plan` and `lineage` do not take a sink per call. A sink is a property of the process rather than of +one migration, and threading it through every call would put it in the signature of every future +operation. + +## Consequences + +The precedent is `Sharder.loader(RouterConfig)`, which already takes the whole configuration to +answer a narrower question. An integrator that has a `RouterConfig` for its router passes the same +one, and the coordinator reads the two observability members from it. + +A coordinator built from a configuration whose observability is unset behaves exactly as one built +from none, because `MetricsHolder` already treats an absent registry and an absent sink as counting +by name and discarding. + +The events a running coordinator emits are still not asserted by the suite, for any surface. This +record does not change that, and +[`../30-conformance.md`](../30-conformance.md#observability-vectors) now says so plainly rather than +claiming an assertion that no driver performs. + +## Alternatives + +A sink on `MigrationPolicy`. Rejected above: the policy is data a plan reports, not a dependency. + +A sink parameter on `plan` and `lineage`. Rejected: it is per process, not per call. + +A coordinator that takes a `Router` and reads its sink. Rejected. It would tie the coordinator's +lifetime to a router's and give it a snapshot it must not read, when what it needs is two members of +a configuration. + +Leaving the coordinator silent and recording the ten events as unimplementable. Rejected. They are +implementable, the contract is published in `OBS-020`, and the withdrawal register makes each name +permanent, so a port that emits nothing has broken a contract rather than deferred one. diff --git a/ports/java/USER-GUIDE.md b/ports/java/USER-GUIDE.md index 2ce407b..f36c4fb 100644 --- a/ports/java/USER-GUIDE.md +++ b/ports/java/USER-GUIDE.md @@ -845,6 +845,11 @@ HandoffCoordinator coordinator = Sharder.coordinator(); MigrationPlan plan = coordinator.plan(before, after, hooks, MigrationPolicy.defaults()); ``` +`Sharder.coordinator(config)` answers a coordinator that reports the `migration.` events of +`OBS-020` through the registry and the sink the configuration carries. It reads those two members +and nothing else from it: no provider, no snapshot, and no health view. +`Sharder.coordinator()` reports nothing, which is the right call where observability is unset. + The coordinator holds nothing between calls and installs nothing. A plan is a pure function of the two snapshots and the policy, which is what lets a restarted coordinator rebuild the same plan. A plan is never created as a side effect of installing a snapshot. diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java index d33bc42..689c227 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java @@ -2,6 +2,7 @@ import com.codeheadsystems.sharder.Router; import com.codeheadsystems.sharder.config.RouterConfig; +import com.codeheadsystems.sharder.core.internal.observe.MetricsHolder; import com.codeheadsystems.sharder.core.internal.route.DefaultRouter; import com.codeheadsystems.sharder.migrate.HandoffCoordinator; import com.codeheadsystems.sharder.migrate.internal.DefaultCoordinator; @@ -34,6 +35,21 @@ public static HandoffCoordinator coordinator() { return new DefaultCoordinator(); } + /** + * The same, reporting through the registry and the sink a configuration carries. + * + *

A coordinator reads the two observability members of the configuration and nothing else + * from it: no provider, no snapshot, and no health view. Where either is unset it reports as + * the sinkless coordinator does, which is to count by name and discard. The events are the + * {@code migration.} rows of {@code OBS-020}, and + * {@code adr/0092} records why the sink arrives this way. + */ + public static HandoffCoordinator coordinator(RouterConfig config) { + return new DefaultCoordinator(new MetricsHolder( + config.observability().metricsRegistry(), config.observability().eventSink()), + config.clock()); + } + /** * A loader that validates a document without installing it, under {@code TOPO-191}. * diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java index f2e1ccf..8b1ca9e 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java @@ -4,6 +4,8 @@ import com.codeheadsystems.sharder.ShardId; import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; +import com.codeheadsystems.sharder.MonotonicClock; +import com.codeheadsystems.sharder.core.internal.observe.MetricsHolder; import com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot; import com.codeheadsystems.sharder.error.InvalidArgumentException; import com.codeheadsystems.sharder.error.PlanRefusedException; @@ -11,11 +13,15 @@ import com.codeheadsystems.sharder.migrate.MigrationPlan; import com.codeheadsystems.sharder.migrate.MigrationPolicy; import com.codeheadsystems.sharder.migrate.LineageClass; +import com.codeheadsystems.sharder.observe.Event; +import com.codeheadsystems.sharder.observe.Severity; import com.codeheadsystems.sharder.migrate.MovementHooks; import com.codeheadsystems.sharder.migrate.ShardLineage; import com.codeheadsystems.sharder.topology.TopologySnapshot; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Where a plan is built, under {@code MOVE-061}. @@ -27,8 +33,23 @@ */ public final class DefaultCoordinator implements HandoffCoordinator { - /** The coordinator every plan is built through. */ + private final MetricsHolder metrics; + private final MonotonicClock clock; + + /** A coordinator that reports nothing, which is what {@code Sharder.coordinator()} answers. */ public DefaultCoordinator() { + this(new MetricsHolder(Optional.empty(), Optional.empty()), MonotonicClock.systemNanoTime()); + } + + /** + * A coordinator reporting through one sink, under {@code adr/0092}. + * + *

The clock stamps the events of {@code OBS-021} and nothing else: a plan reads no clock, + * and {@code step} takes the one the integrator drives it with. + */ + public DefaultCoordinator(MetricsHolder metrics, MonotonicClock clock) { + this.metrics = metrics; + this.clock = clock; } @Override @@ -70,7 +91,11 @@ public MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHo if (needsLocalStep && !hooks.declare().supportsLineage()) { throw new PlanRefusedException(PlanRefusedException.Cause.LINEAGE_UNSUPPORTED); } - return new DefaultMigrationPlan(machine, source, target, hooks, policy); + metrics.emit(new Event("sharder.migration.planned", clock.millis(), source.topologyId(), + target.epoch(), Severity.INFO, + Map.of("handoffCount", Integer.toString(machine.handoffs().size()), + "policy", policy.toString()))); + return new DefaultMigrationPlan(machine, source, target, hooks, policy, metrics, clock); } /** The plan, with a lineage refusal reported as the condition {@code ERR-050} names. */ @@ -100,7 +125,15 @@ public ShardLineage lineage(TopologySnapshot from, TopologySnapshot to) { entries.add(new ShardLineage.Entry(ShardId.of(entry.shard()), LineageClass.valueOf(entry.lineage().name()), parents)); } - return new ShardLineage(entries); + ShardLineage lineage = new ShardLineage(entries); + Map payload = new java.util.LinkedHashMap<>(); + for (LineageClass value : LineageClass.values()) { + payload.put(value.spelling(), + Long.toString(lineage.counts().getOrDefault(value, 0L))); + } + metrics.emit(new Event("sharder.migration.lineage", clock.millis(), source.topologyId(), + target.epoch(), Severity.INFO, payload)); + return lineage; } /** The classification, with a lineage refusal reported as the condition {@code ERR-050} names. */ diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java index 50705f3..5b5999b 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java @@ -5,6 +5,10 @@ import com.codeheadsystems.sharder.core.internal.migrate.Handoff; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; +import com.codeheadsystems.sharder.MonotonicClock; +import com.codeheadsystems.sharder.core.internal.observe.MetricsHolder; +import com.codeheadsystems.sharder.observe.Event; +import com.codeheadsystems.sharder.observe.Severity; import com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot; import com.codeheadsystems.sharder.error.InvalidArgumentException; import com.codeheadsystems.sharder.error.PlanRefusedException; @@ -94,11 +98,17 @@ private static final class Driving { private volatile DocumentSnapshot target; private volatile com.codeheadsystems.sharder.topology.OwnershipDelta delta; + private final MetricsHolder metrics; + // OBS-021 stamps every event with an instant. It is not the clock `step` is driven with: a + // caller advances a plan on its own clock and an event records when it was reported. + private final MonotonicClock eventClock; /** The plan over one machine, its hooks, and the policy that bounds it. */ DefaultMigrationPlan(com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan machine, DocumentSnapshot source, DocumentSnapshot target, MovementHooks hooks, - MigrationPolicy policy) { + MigrationPolicy policy, MetricsHolder metrics, MonotonicClock eventClock) { + this.metrics = metrics; + this.eventClock = eventClock; this.machine = machine; this.source = source; this.target = target; @@ -110,6 +120,12 @@ private static final class Driving { machine.handoffs().forEach(id -> driving.put(id, new Driving())); } + /** One {@code migration.} event of {@code OBS-020}, stamped as {@code OBS-021} requires. */ + private void report(String name, Severity severity, Map payload) { + metrics.emit(new Event("sharder.migration." + name, eventClock.millis(), + source.topologyId(), target.epoch(), severity, payload)); + } + @Override public com.codeheadsystems.sharder.topology.OwnershipDelta delta() { return delta; @@ -170,6 +186,34 @@ public StepOutcome step(MonotonicClock clock) { return new StepOutcome.Idle(); } + /** + * {@code OBS-020}: the transition, and the failure where one is reached. + * + *

{@code migration.state_changed} fires on every transition the machine took, and a + * transition it dropped because the state had moved under a hook is not one, so nothing is + * reported for it. A handoff reaching {@code failed} carries its kind, which is what an + * operator acts on. + */ + private void reportTransition(String id, + String trigger, + com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.StepOutcome outcome) { + if (!"advanced".equals(outcome.outcome()) && !"settled".equals(outcome.outcome())) { + return; + } + report("state_changed", Severity.INFO, Map.of("handoff", id, + "shard", machine.handoff(id).shard(), + "from", outcome.fromState(), + "to", outcome.toState(), + "trigger", trigger)); + if (HandoffState.FAILED.spelling().equals(outcome.toState())) { + Handoff handoff = machine.handoff(id); + report("failed", Severity.ERROR, Map.of("shard", handoff.shard(), + "kind", handoff.failureKind().orElse("unknown"), + "source", handoff.source().asText(), + "destination", handoff.destination().asText())); + } + } + /** * One transition of the machine, taken under its lock. * @@ -188,7 +232,9 @@ private com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.StepOutc synchronized (machineLock) { HandoffState from = machine.handoff(id).state(); try { - return machine.step(id, trigger, at, pressure); + var outcome = machine.step(id, trigger, at, pressure); + reportTransition(id, trigger, outcome); + return outcome; } catch (IllegalStateException moved) { // The state named no transition for this trigger, which is what a concurrent // abort or recovery leaves behind. @@ -363,11 +409,27 @@ private StepOutcome commit(String id, Handoff handoff, Driving state, long at) { if ("idle".equals(outcome.outcome())) { // MOVE-333: the commit deadline ran past the horizon, so the lease is spent and // MOVE-331 takes a fresh quiesce on the next step. + // An idle outcome here is the spent lease where the machine says so, and a + // concurrent abort otherwise, which is not this event. + if ("quiesceExpired".equals(outcome.reason())) { + report("quiesce_expired", Severity.ERROR, Map.of("handoff", id, + "shard", handoff.shard(), + "leaseMillis", Long.toString(handoff.leaseMillis()), + "marginMillis", Long.toString(policy.quiesceLeaseMarginMillis()))); + } synchronized (machineLock) { machine.clearQuiesce(id); } return new StepOutcome.Progressed(HandoffId.of(id), 0); } + // MOVE-311: the window is the interval the shard was quiesced for, from the reading + // `MOVE-332` took before the commit to the reading this step was driven with. + report("cutover_committed", Severity.INFO, Map.of("shard", handoff.shard(), + "source", handoff.source().asText(), + "destination", handoff.destination().asText(), + "windowMillis", Long.toString( + handoff.quiesceInstant().isPresent() + ? at - handoff.quiesceInstant().getAsLong() : 0L))); return advanced(id, handoff.state(), outcome); } @@ -539,7 +601,7 @@ public ReobserveOutcome reobserve(HandoffId id, MonotonicClock clock) { // MOVE-233 re-observes a terminal handoff; a live one is advanced by `step`. return new ReobserveOutcome.Refused("the handoff is not terminal"); } - return switch (hooks.observe(context(handoff, state))) { + ReobserveOutcome outcome = switch (hooks.observe(context(handoff, state))) { case ObserveResult.Observed observed -> { resume(id.value(), handoff, observed.observation()); yield new ReobserveOutcome.Resumed(id, @@ -548,6 +610,15 @@ public ReobserveOutcome reobserve(HandoffId id, MonotonicClock clock) { case ObserveResult.Unavailable unavailable -> new ReobserveOutcome.Unresolved(); case ObserveResult.Undetermined undetermined -> new ReobserveOutcome.Unresolved(); }; + // OBS-020: `answer` is what the hook established, and `resumedState` the state the + // handoff is in afterwards, which is unchanged where nothing was established. + report("reobserved", Severity.INFO, Map.of("handoff", id.value(), + "shard", handoff.shard(), + "answer", outcome instanceof ReobserveOutcome.Resumed ? "observed" + : outcome instanceof ReobserveOutcome.Unresolved ? "unresolved" + : "refused", + "resumedState", machine.handoff(id.value()).state().spelling())); + return outcome; } finally { state.claimed.set(false); } @@ -622,8 +693,15 @@ public RebaseReport rebase(TopologySnapshot to) { var report = machine.rebase(newer.epoch(), replicaSets); target = newer; delta = com.codeheadsystems.sharder.topology.OwnershipDelta.between(source, newer); - return new RebaseReport(report.fromEpoch(), report.toEpoch(), + RebaseReport answer = new RebaseReport(report.fromEpoch(), report.toEpoch(), ids(report.rebased()), ids(report.aborted()), ids(report.unchanged())); + report("rebased", Severity.INFO, Map.of( + "fromEpoch", Long.toString(answer.fromEpoch()), + "toEpoch", Long.toString(answer.toEpoch()), + "rebased", Integer.toString(answer.rebased().size()), + "aborted", Integer.toString(answer.aborted().size()), + "unchanged", Integer.toString(answer.unchanged().size()))); + return answer; } } finally { claims.forEach(claim -> claim.claimed.set(false)); @@ -697,12 +775,25 @@ public void onSnapshotInstalled(TopologySnapshot snapshot) { if (!(snapshot instanceof DocumentSnapshot installed)) { throw new InvalidArgumentException("the snapshot was not produced by this library"); } + com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.InstallOutcome outcome; synchronized (machineLock) { // MOVE-092: the mark records the installed snapshot and nothing else. No hook is // called, no preference list is evaluated, and no delta is computed. - machine.onSnapshotInstalled(installed.topologyId(), installed.epoch(), + outcome = machine.onSnapshotInstalled(installed.topologyId(), installed.epoch(), source.topologyId()); } + if (outcome.superseded()) { + report("superseded", Severity.WARNING, Map.of( + "installedEpoch", Long.toString(installed.epoch()), + "abortedCount", Integer.toString(outcome.aborted().size()), + "finishingCount", Integer.toString(outcome.finishing().size()))); + } + if (outcome.rebasePending() != null) { + // MOVE-091: the plan is marked and the integrator decides whether to rebase it. + report("rebase_pending", Severity.WARNING, Map.of( + "installedEpoch", Long.toString(installed.epoch()), + "targetEpoch", Long.toString(outcome.rebasePending()))); + } } private static List ids(List named) { diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java b/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java index cbe1b82..20f6306 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java @@ -15,7 +15,9 @@ import com.codeheadsystems.sharder.migrate.HookDeclaration; import com.codeheadsystems.sharder.migrate.HookResult; import com.codeheadsystems.sharder.migrate.MigrationPlan; +import com.codeheadsystems.sharder.migrate.HandoffCoordinator; import com.codeheadsystems.sharder.migrate.MigrationPolicy; +import com.codeheadsystems.sharder.observe.Event; import com.codeheadsystems.sharder.NodeId; import com.codeheadsystems.sharder.ShardId; import com.codeheadsystems.sharder.error.InvalidArgumentException; @@ -83,6 +85,60 @@ private static byte[] slots(long epoch, String... owners) { return json.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); } + /** + * {@code OBS-020}: the `migration.` events a coordinator reports, under {@code adr/0092}. + * + *

The suite asserts no emitted event for any surface, so this is where the port's own + * emission is checked. It drives one handoff from `planned` to `complete` and asserts the + * events that run produces, their severities, and the common members of {@code OBS-021}. + */ + @Test + void aCoordinatorReportsTheEventsOfItsSurface() { + Pair pair = snapshots(slots(1, "a", "b"), slots(2, "a", "c")); + List events = new java.util.concurrent.CopyOnWriteArrayList<>(); + RouterConfig config = RouterConfig.builder() + .provider(new InMemoryTopologyProvider(slots(1, "a", "b"))) + .clock(CLOCK) + .eventSink(events::add) + .build(); + HandoffCoordinator coordinator = Sharder.coordinator(config); + + coordinator.lineage(pair.from(), pair.to()); + MigrationPlan plan = coordinator.plan(pair.from(), pair.to(), new RecordingHooks(), + MigrationPolicy.defaults()); + for (int step = 0; step < 12; step++) { + if (plan.step(CLOCK) instanceof StepOutcome.Settled) { + break; + } + } + + List names = events.stream().map(Event::name).distinct().toList(); + assertThat(names) + .contains("sharder.migration.lineage", "sharder.migration.planned", + "sharder.migration.state_changed", "sharder.migration.cutover_committed"); + // OBS-021: every event carries the topology and epoch in force and a severity. + assertThat(events).allSatisfy(event -> { + assertThat(event.topologyId()).isEqualTo("slots"); + assertThat(event.epoch()).isEqualTo(2L); + assertThat(event.severity()).isNotNull(); + assertThat(event.payload()).isNotEmpty(); + }); + // Every payload member the inventory names for an emitted event is carried. + Event planned = events.stream() + .filter(event -> event.name().equals("sharder.migration.planned")) + .findFirst().orElseThrow(); + assertThat(planned.payload()).containsKeys("handoffCount", "policy"); + Event lineage = events.stream() + .filter(event -> event.name().equals("sharder.migration.lineage")) + .findFirst().orElseThrow(); + assertThat(lineage.payload()).containsKeys("unchanged", "moved", "divided", "merged", + "fresh", "split", "folded", "vacated"); + Event changed = events.stream() + .filter(event -> event.name().equals("sharder.migration.state_changed")) + .findFirst().orElseThrow(); + assertThat(changed.payload()).containsKeys("handoff", "shard", "from", "to", "trigger"); + } + @Test void oneHandoffRunsFromPlannedToComplete() { Pair pair = snapshots(slots(1, "a", "b"), slots(2, "a", "c"));