diff --git a/conformance/coverage.json b/conformance/coverage.json index 78f46d8..57b9b0a 100644 --- a/conformance/coverage.json +++ b/conformance/coverage.json @@ -1,11 +1,33 @@ { "specification": "docs/design/10-specification.md", - "statedRequirements": 656, - "coveredRequirements": 454, - "uncoveredRequirements": 202, - "coveragePercent": 69, + "statedRequirements": 689, + "coveredRequirements": 474, + "uncoveredRequirements": 215, + "coveragePercent": 68, "identifiersNamedButNotStated": [], "byPrefix": [ + { + "prefix": "LIN", + "section": "", + "stated": 33, + "covered": 20, + "uncovered": [ + "LIN-001", + "LIN-002", + "LIN-003", + "LIN-005", + "LIN-015", + "LIN-023", + "LIN-032", + "LIN-034", + "LIN-043", + "LIN-053", + "LIN-054", + "LIN-055", + "LIN-058" + ], + "percent": 60 + }, { "prefix": "CFG", "section": "Configuration surface", @@ -735,6 +757,7 @@ "REPL-021", "REPL-025", "RING-013", + "RING-031", "RV-013", "SEC-011", "SEC-013", @@ -769,7 +792,7 @@ "TOPO-231", "TOPO-241" ], - "requirementsAtLevel": 125, + "requirementsAtLevel": 126, "requirementsWhenDeclared": 284 }, { @@ -957,6 +980,8 @@ "CFG-050", "CFG-052", "CFG-055", + "DIR-002", + "DIR-010", "ERR-050", "ERR-052", "ERR-053", @@ -964,6 +989,26 @@ "HEALTH-041", "HEALTH-043", "HEALTH-048", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-016", + "LIN-021", + "LIN-022", + "LIN-031", + "LIN-033", + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-056", + "LIN-057", "MOVE-001", "MOVE-011", "MOVE-021", @@ -1020,6 +1065,7 @@ "OBS-010", "OBS-020", "OBS-021", + "PLACE-065", "RATE-001", "RATE-011", "RATE-021", @@ -1035,8 +1081,8 @@ "TOPO-221", "TOPO-231" ], - "requirementsAtLevel": 80, - "requirementsWhenDeclared": 404 + "requirementsAtLevel": 103, + "requirementsWhenDeclared": 424 } ], "coveredBy": { @@ -1247,7 +1293,8 @@ "RING-031": [ "vectors/determinism/tie-breaks.json", "vectors/placement/shard-enumeration.json", - "vectors/scale/ring-1000.json" + "vectors/scale/ring-1000.json", + "vectors/topology/ownership-delta.json" ], "RV-003": [ "vectors/determinism/tie-breaks.json", @@ -1270,6 +1317,7 @@ ], "TOPO-061": [ "properties/properties.json", + "scenarios/topology-acceptance-floor.json", "scenarios/topology-rollback.json", "vectors/digest/canonical-form.json" ], @@ -1277,7 +1325,8 @@ "vectors/directory/matcher-precedence.json" ], "DIR-002": [ - "vectors/directory/matcher-precedence.json" + "vectors/directory/matcher-precedence.json", + "vectors/migration/lineage.json" ], "DIR-003": [ "vectors/directory/matcher-precedence.json", @@ -1315,11 +1364,13 @@ ], "PLACE-065": [ "vectors/directory/matcher-precedence.json", + "vectors/migration/lineage.json", "vectors/overrides/precedence-and-composition.json", "vectors/rendezvous/tenant-router.json" ], "DIR-010": [ - "vectors/directory/no-match.json" + "vectors/directory/no-match.json", + "vectors/migration/lineage.json" ], "DIR-011": [ "vectors/directory/no-match.json" @@ -1482,7 +1533,8 @@ ], "ERR-050": [ "scenarios/rebase-drops-a-handoff.json", - "vectors/formulas/migration-rate.json" + "vectors/formulas/migration-rate.json", + "vectors/migration/lineage.json" ], "MOVE-171": [ "properties/properties.json", @@ -1631,6 +1683,80 @@ "KEY-044": [ "vectors/keytransform/prefix-fields-2.json" ], + "LIN-004": [ + "vectors/migration/lineage.json" + ], + "LIN-006": [ + "vectors/migration/lineage.json" + ], + "LIN-007": [ + "vectors/migration/lineage.json" + ], + "LIN-011": [ + "vectors/migration/lineage.json" + ], + "LIN-012": [ + "vectors/migration/lineage.json" + ], + "LIN-013": [ + "vectors/migration/lineage.json" + ], + "LIN-014": [ + "vectors/migration/lineage.json" + ], + "LIN-016": [ + "vectors/migration/lineage.json" + ], + "LIN-021": [ + "vectors/migration/lineage.json" + ], + "LIN-022": [ + "vectors/migration/lineage.json" + ], + "LIN-031": [ + "vectors/migration/lineage.json" + ], + "LIN-033": [ + "vectors/migration/lineage.json" + ], + "MOVE-241": [ + "scenarios/handoff-happy-path.json", + "vectors/migration/lineage.json", + "vectors/topology/ownership-delta-unsupported.json" + ], + "TOPO-231": [ + "scenarios/plan-superseded-by-new-epoch.json", + "scenarios/rebase-drops-a-handoff.json", + "vectors/migration/lineage.json", + "vectors/topology/ownership-delta.json" + ], + "LIN-041": [ + "vectors/migration/plan-construction.json" + ], + "LIN-042": [ + "vectors/migration/plan-construction.json" + ], + "LIN-044": [ + "vectors/migration/plan-construction.json" + ], + "LIN-045": [ + "vectors/migration/plan-construction.json" + ], + "LIN-051": [ + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "vectors/migration/plan-construction.json" + ], + "LIN-052": [ + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "vectors/migration/plan-construction.json" + ], + "LIN-057": [ + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "vectors/migration/plan-construction.json" + ], "PROP-010": [ "properties/properties.json", "vectors/movement/add-one-node.json" @@ -2229,10 +2355,6 @@ "vectors/spread/degradation-ladder.json", "vectors/spread/relaxation-stages.json" ], - "MOVE-241": [ - "scenarios/handoff-happy-path.json", - "vectors/topology/ownership-delta-unsupported.json" - ], "MOVE-251": [ "vectors/topology/ownership-delta-unsupported.json" ], @@ -2255,11 +2377,6 @@ "scenarios/handoff-happy-path.json", "vectors/topology/ownership-delta.json" ], - "TOPO-231": [ - "scenarios/plan-superseded-by-new-epoch.json", - "scenarios/rebase-drops-a-handoff.json", - "vectors/topology/ownership-delta.json" - ], "TOPO-241": [ "vectors/topology/ownership-delta.json" ], @@ -2364,7 +2481,8 @@ "scenarios/topology-rollback.json" ], "TOPO-071": [ - "properties/properties.json" + "properties/properties.json", + "scenarios/topology-acceptance-floor.json" ], "TOPO-081": [ "properties/properties.json", @@ -2376,6 +2494,7 @@ ], "ERR-031": [ "properties/properties.json", + "scenarios/topology-acceptance-floor.json", "scenarios/topology-rollback.json" ], "ERR-034": [ @@ -2417,6 +2536,9 @@ "scenarios/abort-during-catching-up.json", "scenarios/handoff-failure-kinds.json", "scenarios/handoff-happy-path.json", + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", + "scenarios/handoff-local-step-aborted.json", "scenarios/node-dies-mid-migration.json", "scenarios/quiesce-lease-expiry.json", "scenarios/undetermined-resolves-both-ways.json" @@ -2425,6 +2547,8 @@ "properties/properties.json", "scenarios/handoff-failure-kinds.json", "scenarios/handoff-happy-path.json", + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json", "scenarios/undetermined-resolves-both-ways.json" ], "MOVE-441": [ @@ -2477,6 +2601,7 @@ "properties/properties.json" ], "ERR-032": [ + "scenarios/topology-acceptance-floor.json", "scenarios/topology-rollback.json" ], "ERR-040": [ @@ -2570,7 +2695,9 @@ "scenarios/redirect-walk-depth-limit.json" ], "MOVE-001": [ - "scenarios/handoff-happy-path.json" + "scenarios/handoff-happy-path.json", + "scenarios/handoff-local-division.json", + "scenarios/handoff-local-fold.json" ], "MOVE-041": [ "scenarios/handoff-happy-path.json" @@ -2605,6 +2732,15 @@ "scenarios/handoff-happy-path.json", "scenarios/quiesce-lease-expiry.json" ], + "LIN-056": [ + "scenarios/handoff-local-step-aborted.json" + ], + "MOVE-421": [ + "scenarios/abort-during-catching-up.json", + "scenarios/handoff-local-step-aborted.json", + "scenarios/node-dies-mid-migration.json", + "scenarios/rebase-drops-a-handoff.json" + ], "CFG-055": [ "scenarios/quiesce-lease-expiry.json" ], @@ -2634,11 +2770,6 @@ "HEALTH-048": [ "scenarios/node-dies-mid-migration.json" ], - "MOVE-421": [ - "scenarios/abort-during-catching-up.json", - "scenarios/node-dies-mid-migration.json", - "scenarios/rebase-drops-a-handoff.json" - ], "MOVE-481": [ "scenarios/node-dies-mid-migration.json", "scenarios/plan-superseded-by-new-epoch.json", diff --git a/conformance/declarations/java.json b/conformance/declarations/java.json index c9a4ec9..d10bbbd 100644 --- a/conformance/declarations/java.json +++ b/conformance/declarations/java.json @@ -1,7 +1,7 @@ { "port": "java", "version": "unreleased", - "revision": "e91417161ae0de0002cf3748a9ecd969e82c6233f0d8ab97fd3e908492b24204", + "revision": "c29e6a8f3facafa56414e46851ada1610db5e64c016714a8dba915e08875d57c", "strategySurfaces": [ "directory", "rendezvous", @@ -37,13 +37,13 @@ "run": { "command": "cd ports/java && ./gradlew test", "report": "ports/java/conformance/report.txt", - "vectorFiles": 67, - "vectorCases": 596, + "vectorFiles": 69, + "vectorCases": 609, "failures": 0 }, "scale": { - "wallTimeMs": 833, - "peakResidentBytes": 683659264, + "wallTimeMs": 869, + "peakResidentBytes": 614211584, "machine": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics, Linux 6.8.0-138-generic", "runtime": "OpenJDK 64-Bit Server VM 25.0.4, compiled against the Java 21 API" }, diff --git a/conformance/driver/python/run_suite.py b/conformance/driver/python/run_suite.py index 97d97b0..b60796c 100644 --- a/conformance/driver/python/run_suite.py +++ b/conformance/driver/python/run_suite.py @@ -389,6 +389,36 @@ def run_ownership_delta(root, payload): compare(case["name"] + ".delta", rows, case["expect"]["delta"]) +def run_lineage(root, payload): + from sharder_ref import lineage + from sharder_ref.handoff import replica_set + for case in payload["cases"]: + before = load_topology(root, payload, case["before"]) + after = load_topology(root, payload, case["after"]) + replicas = lambda s, shard: replica_set(s, shard, lambda x: x.factor) + if case["expect"].get("lineageComputed") is False: + try: + lineage.classify(before, after, replicas) + except lineage.Refused as refusal: + compare(case["name"] + ".cause", refusal.cause, + case["expect"]["condition"]["cause"]) + continue + raise AssertionError("%s: a lineage was computed where one is refused" % case["name"]) + rows = lineage.classify(before, after, replicas) + compare(case["name"] + ".lineage", rows, case["expect"]["lineage"]) + + +def run_plan_construction(root, payload): + from sharder_ref import lineage + from sharder_ref.handoff import replica_set + for case in payload["cases"]: + before = load_topology(root, payload, case["before"]) + after = load_topology(root, payload, case["after"]) + replicas = lambda s, shard: replica_set(s, shard, lambda x: x.factor) + handoffs = lineage.plan_handoffs(before, after, replicas) + compare(case["name"] + ".handoffs", handoffs, case["expect"]["handoffs"]) + + def run_pin_shard(root, payload): for case in payload["cases"]: snapshot = load_topology(root, payload, payload["topology"]) @@ -544,6 +574,8 @@ def run_publication_events(root, payload): "errorTaxonomy": run_error_taxonomy, "defaults": run_defaults, "ownershipDelta": run_ownership_delta, + "lineage": run_lineage, + "planConstruction": run_plan_construction, "pinShard": run_pin_shard, "readAffinity": run_read_affinity, "propertyWitness": run_property_witness, @@ -605,6 +637,7 @@ def run_scenarios(root, levels, known, verbose): scenario = json.loads((root / entry["file"]).read_text()) retained = {} in_force = None + loader_setup = scenario.get("setup", {}).get("loader", {}) for step in scenario["steps"]: action = step["action"] try: @@ -616,15 +649,25 @@ def run_scenarios(root, levels, known, verbose): compare(action + ".digest", jcs_digest(document), step["expect"]["digest"]) candidate = Snapshot(document) - outcome, condition = accept(candidate, in_force) + # `TOPO-061` and `TOPO-071`: a scenario states the identifier and the epoch + # floor its process was configured with, and both are checked before the row + # that installs a first document. + outcome, condition = accept( + candidate, in_force, + min_epoch=loader_setup.get("minEpoch"), + expected_topology_id=loader_setup.get("expectedTopologyId")) compare(action + ".outcome", outcome, step["expect"]["outcome"]) compare(action + ".condition", condition, step["expect"]["condition"]) if outcome == "installed": if in_force is not None: retained[in_force.epoch] = in_force in_force = candidate - compare(action + ".epochInForce", in_force.epoch, - step["expect"]["epochInForce"]) + # A scenario whose first document is refused has nothing in force, so the + # expectation states no epoch and there is none to compare. + if "epochInForce" in step["expect"]: + compare(action + ".epochInForce", + None if in_force is None else in_force.epoch, + step["expect"]["epochInForce"]) checked += 1 elif action == "recipientCheck": snapshot = None if step.get("noSnapshot") else in_force diff --git a/conformance/generator/generate_extra.py b/conformance/generator/generate_extra.py index 9c3746e..f52bdbd 100644 --- a/conformance/generator/generate_extra.py +++ b/conformance/generator/generate_extra.py @@ -10,6 +10,7 @@ import argparse import copy +import json import sys from pathlib import Path @@ -18,7 +19,7 @@ import topologies as T # noqa: E402 from generate import key_spec, write_json # noqa: E402 -from sharder_ref import handoff, placement, routing # noqa: E402 +from sharder_ref import handoff, lineage, placement, routing # noqa: E402 from sharder_ref.topology import Snapshot # noqa: E402 ENTRIES = [] @@ -75,13 +76,14 @@ def emit(root, path, vector_set, kind, description, requirements, cases, topolog (401, "planRefused", "no", "a plan cannot be built from the two snapshots and the policy", "correct the snapshots or the policy member named in cause", ["incomparableShards", "epochNotAdvancing", "strategyUnsupported", - "destinationOutsidePlacementSet", "policyInvalid", "topologyMismatch"]), + "destinationOutsidePlacementSet", "policyInvalid", "topologyMismatch", "unalignedLineage", + "lineageUnsupported"]), (402, "quiesced", "yes", "the shard is inside the cutover window", "retry after the window, which commitDeadlineMillis bounds", []), (403, "handoffFailed", "no", "a handoff reached failed", "operator action, directed by the failure kind in cause; undetermined takes a" " re-observation", - ["unverified", "residue", "undetermined", "rollbackFailed"]), + ["unverified", "residue", "undetermined", "rollbackFailed", "undivided"]), ] @@ -380,12 +382,201 @@ def build_identity_comparison(root): "nodes": ["d", "b", "c", "a"]}] +# `TOPO-213`: a `ring` pair whose two snapshots enumerate different shard sets. Adding the token +# `0000000000002000` divides the extent the token `0000000000003000` bounded, so the later snapshot +# enumerates a shard the earlier one does not; removing it folds that extent back, so the earlier +# snapshot enumerates one the later one does not. Every other pair in this file is `slot` at a +# fixed `slotCount`, where the two shard sets are always equal and the second clause of `TOPO-213` +# is unreachable. +DELTA_RING_BEFORE = { + "formatVersion": "1.0", "topologyId": "delta-ring", "epoch": 1, + "replication": {"factor": 2}, + "strategy": {"kind": "ring", "tokenAssignment": "explicit"}, + "nodes": [{"id": "a", "tokens": ["0000000000001000", "0000000000005000"]}, + {"id": "b", "tokens": ["0000000000003000", "0000000000007000"]}], +} + +DELTA_RING_ADDED = copy.deepcopy(DELTA_RING_BEFORE) +DELTA_RING_ADDED["epoch"] = 2 +DELTA_RING_ADDED["nodes"].append({"id": "c", "tokens": ["0000000000002000"]}) + +DELTA_RING_REMOVED = copy.deepcopy(DELTA_RING_BEFORE) +DELTA_RING_REMOVED["epoch"] = 3 + + +# `LIN-022`: removing the token `0000000000003000` and adding `0000000000002000` in the same epoch +# moves a boundary without dividing or folding an extent whole. The extent `(2000, 5000]` of the +# later snapshot overlaps `(1000, 3000]` of the earlier one and neither contains the other, so the +# lineage is unaligned and the plan is refused. +DELTA_RING_UNALIGNED = copy.deepcopy(DELTA_RING_BEFORE) +DELTA_RING_UNALIGNED["epoch"] = 4 +DELTA_RING_UNALIGNED["nodes"] = [{"id": "a", "tokens": ["0000000000001000", "0000000000005000"]}, + {"id": "b", "tokens": ["0000000000007000"]}, + {"id": "c", "tokens": ["0000000000002000"]}] + +# `LIN-013`: a `directory` pair whose entry sets differ. A directory extent is a matcher narrowed +# by the precedence of `DIR-002`, which is decidable and not yet defined, so the pair is refused. +LINEAGE_DIR_BEFORE = { + "formatVersion": "1.0", "topologyId": "lineage-directory", "epoch": 1, + "replication": {"factor": 1}, + "strategy": {"kind": "directory", + "entries": [{"match": {"kind": "prefix", "value": "ab"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "cd"}, "nodes": ["d2"]}]}, + "nodes": [{"id": "d1"}, {"id": "d2"}], +} + +# `LIN-043`: `ef` matched no entry of the earlier table, so the shard it names has no parent and +# no contents to move. A `directory` table is the only place a fresh extent arises, because it is +# the only kind whose `shardOf` answers with no shard under `DIR-010`. +LINEAGE_DIR_FRESH = copy.deepcopy(LINEAGE_DIR_BEFORE) +LINEAGE_DIR_FRESH["epoch"] = 3 +LINEAGE_DIR_FRESH["strategy"]["entries"] = [ + {"match": {"kind": "prefix", "value": "ab"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "cd"}, "nodes": ["d2"]}, + {"match": {"kind": "prefix", "value": "ef"}, "nodes": ["d1"]}, +] + +LINEAGE_DIR_REFINED = copy.deepcopy(LINEAGE_DIR_BEFORE) +LINEAGE_DIR_REFINED["epoch"] = 2 +LINEAGE_DIR_REFINED["strategy"]["entries"] = [ + {"match": {"kind": "prefix", "value": "ab0"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "ab1"}, "nodes": ["d2"]}, + {"match": {"kind": "prefix", "value": "ab"}, "nodes": ["d1"]}, + {"match": {"kind": "prefix", "value": "cd"}, "nodes": ["d2"]}, +] + + +def build_lineage(root): + """`LIN-*`: the classification over pairs of documents, and the plan built over it.""" + documents = {"lineage-directory-before": LINEAGE_DIR_BEFORE, + "lineage-directory-refined": LINEAGE_DIR_REFINED, + "lineage-directory-fresh": LINEAGE_DIR_FRESH, + "delta-ring-unaligned": DELTA_RING_UNALIGNED} + for name, document in documents.items(): + write_json(root / ("topologies/%s.topology.json" % name), document) + + snap = {name: Snapshot(d) for name, d in documents.items()} + snap.update({name: Snapshot(d) for name, d in + {"delta-ring-before": DELTA_RING_BEFORE, "delta-ring-added": DELTA_RING_ADDED, + "delta-ring-removed": DELTA_RING_REMOVED, "delta-before": DELTA_BEFORE, + "delta-moved": DELTA_MOVED, + "delta-other-seed": DELTA_OTHER_SEED}.items()}) + snap["rendezvous-plain"] = Snapshot( + json.loads((root / "topologies/rendezvous-plain.topology.json").read_text())) + + def replicas(snapshot, shard): + return handoff.replica_set(snapshot, shard, lambda s: s.factor) + + def path(name): + return "topologies/%s.topology.json" % name + + cases = [] + for label, before, after, requirements, note in [ + ("ring-extent-divided", "delta-ring-before", "delta-ring-added", + ["LIN-004", "LIN-011", "LIN-021", "LIN-031", "LIN-033"], + "`LIN-011`: the added token divides `(1000, 3000]` into `(1000, 2000]` and " + "`(2000, 3000]`, so both shards of the later snapshot are `divided` from one parent and " + "the shard whose extent did not move is `moved` because its replica set did."), + ("ring-extent-folded", "delta-ring-added", "delta-ring-removed", + ["LIN-004", "LIN-011", "LIN-021", "LIN-033"], + "`LIN-021`: removing the token folds `(1000, 2000]` into the extent that follows it, so " + "the later shard is `merged` from two parents and the shard only the earlier snapshot " + "enumerates is `folded` and follows every later entry under `LIN-033`."), + ("directory-prefix-refined", "lineage-directory-before", "lineage-directory-refined", + ["LIN-013", "LIN-016", "LIN-021", "DIR-002", "PLACE-065"], + "`LIN-016`: refining `prefix:ab` into `ab0` and `ab1` leaves `ab` winning the keys " + "neither longer prefix claims, so all three shards of the later table are `divided` from " + "the one entry and the untouched `cd` is `unchanged`."), + ("directory-fresh-extent", "lineage-directory-before", "lineage-directory-fresh", + ["LIN-013", "LIN-016", "LIN-021", "DIR-010"], + "`LIN-021`: the added entry wins keys the earlier table matched to no shard under " + "`DIR-010`, so the shard it names is `fresh` and has no parent to draw from."), + ("slot-identity", "delta-before", "delta-moved", + ["LIN-007", "LIN-012", "LIN-021"], + "`LIN-012`: `TOPO-231` holds `slotCount` equal, so the two snapshots enumerate the same " + "shards, the lineage is the identity of `LIN-007`, and a shard is `moved` or `unchanged` " + "and never divided, merged, fresh, or vacated."), + ]: + cases.append({ + "name": label, "requirements": requirements, + "before": path(before), "after": path(after), "note": note, + "expect": {"lineage": lineage.classify(snap[before], snap[after], replicas)}, + }) + + for label, before, after, requirements, cause, note in [ + ("ring-unaligned-boundary", "delta-ring-before", "delta-ring-unaligned", + ["LIN-021", "LIN-022", "ERR-050"], "unalignedLineage", + "`LIN-022`: a boundary that moves without dividing or folding an extent whole has no " + "correspondence to name, so the plan is refused and the authority publishes the change " + "as a division epoch followed by a fold epoch."), + ("incomparable-shard-identity", "delta-before", "delta-other-seed", + ["LIN-006", "TOPO-231", "ERR-050"], "incomparableShards", + "`LIN-006`: a lineage joins two snapshots on their extents and an ownership delta joins " + "them on their identifiers, and a change that renames every shard leaves neither one an " + "answer, so both refuse on the condition `TOPO-231` states."), + ("rendezvous-has-no-extent", "rendezvous-plain", "rendezvous-plain", + ["LIN-014", "MOVE-241", "ERR-050"], "strategyUnsupported", + "`LIN-014`: `PLACE-032` makes `shards` empty under `rendezvous`, so the kind has no " + "extent and no lineage, and `MOVE-251` refuses the plan before one is reached."), + ]: + cases.append({ + "name": label, "requirements": requirements, + "before": path(before), "after": path(after), "note": note, + "expect": {"lineageComputed": False, + "condition": {"code": 401, "name": "planRefused", "cause": cause}}, + }) + + emit(root, "vectors/migration/lineage.json", "migration-lineage", "lineage", + "The lineage classification over pairs of documents: a ring extent divided, a ring extent " + "folded, the identity lineage under `slot`, the unaligned boundary that is refused, the " + "`directory` prefix refined and the fresh extent beside it, the incomparable pair, and " + "the kind that enumerates no shard.", + ["LIN-004", "LIN-006", "LIN-007", "LIN-011", "LIN-012", "LIN-013", "LIN-014", "LIN-016", + "LIN-021", "LIN-022", "LIN-031", "LIN-033", "DIR-002", "DIR-010", "PLACE-065", + "TOPO-231", "MOVE-241", "ERR-050"], cases, level="migration") + + plans = [] + for label, before, after, requirements, note in [ + ("divided-source-from-the-parent", "delta-ring-before", "delta-ring-added", + ["LIN-041", "LIN-042", "LIN-045", "LIN-051", "LIN-052", "LIN-057"], + "`LIN-041`: the contents of the divided shard `0000000000002000` are held by the " + "replicas of its parent `0000000000003000`, so the handoff names one of them as its " + "source. A plan built over the ownership delta alone has no entry for the parent, whose " + "replica set did not change, and names a source equal to the destination."), + ("folded-destination-outside-the-delta", "delta-ring-added", "delta-ring-removed", + ["LIN-041", "LIN-044", "LIN-045", "LIN-051", "LIN-052", "LIN-057"], + "`LIN-044`: the shard that absorbs the folded extent keeps its replica set, so the " + "ownership delta reports no entry for it, and a plan built over the delta alone moves " + "nothing to the replica that does not hold the folded parent."), + ("slot-plan-over-the-identity-lineage", "delta-before", "delta-moved", + ["LIN-041", "LIN-045"], + "`LIN-007`: under the identity lineage every shard is its own parent, so the plan is the " + "one the ownership delta already produced and this case is the regression guard for it."), + ]: + plans.append({ + "name": label, "requirements": requirements, + "before": path(before), "after": path(after), "note": note, + "expect": {"handoffs": lineage.plan_handoffs(snap[before], snap[after], replicas)}, + }) + + emit(root, "vectors/migration/plan-construction.json", "migration-plan-construction", + "planConstruction", + "How a plan derives a handoff from a lineage: the source of a divided shard, the " + "destination of a fold that the ownership delta does not report, the local steps beside " + "them and their order, and the plan under the identity lineage.", + ["LIN-041", "LIN-042", "LIN-044", "LIN-045", "LIN-051", "LIN-052", "LIN-057"], plans, + level="migration") + + + def build_ownership_delta(root): documents = {"delta-before": DELTA_BEFORE, "delta-moved": DELTA_MOVED, "delta-reordered": DELTA_REORDERED, "delta-other-slot-count": DELTA_OTHER_COUNT, "delta-other-seed": DELTA_OTHER_SEED, "delta-wide-before": DELTA_WIDE_BEFORE, "delta-wide-after": DELTA_WIDE_AFTER, - "delta-short-before": DELTA_SHORT_BEFORE, "delta-short-after": DELTA_SHORT_AFTER} + "delta-short-before": DELTA_SHORT_BEFORE, "delta-short-after": DELTA_SHORT_AFTER, + "delta-ring-before": DELTA_RING_BEFORE, "delta-ring-added": DELTA_RING_ADDED, + "delta-ring-removed": DELTA_RING_REMOVED} for name, document in documents.items(): write_json(root / ("topologies/%s.topology.json" % name), document) snapshots = {name: Snapshot(d) for name, d in documents.items()} @@ -429,6 +620,16 @@ def build_ownership_delta(root): "`TOPO-213`: the entries follow the ascending slot index `SLOT-031` enumerates, so slots " "8 and 9 precede slot 10. Ordering the shard identifiers as octets would put 10 first " "and is the divergence a `slotCount` at or below 10 cannot show."), + ("ring-token-added", "delta-ring-before", "delta-ring-added", + ["TOPO-211", "TOPO-213", "PLACE-031", "RING-031"], + "`TOPO-213`: the added token divides the extent `0000000000003000` bounded, so the " + "second snapshot enumerates `0000000000002000` and the first does not. That shard's " + "entry carries an empty before set and reports every node it gained."), + ("ring-token-removed", "delta-ring-added", "delta-ring-removed", + ["TOPO-211", "TOPO-213", "PLACE-031", "RING-031"], + "`TOPO-213`: removing the token folds `0000000000002000` into the extent that follows " + "it, so only the first snapshot enumerates that shard. Its entry carries an empty " + "after set and follows every entry for a shard the second snapshot enumerates."), ("replica-prefix-short-of-factor", "delta-short-before", "delta-short-after", ["TOPO-211", "REPL-017", "REPL-020", "SPREAD-014"], "`TOPO-211`: `strict` over one zone level admits two replicas of the three the factor " @@ -448,10 +649,11 @@ def build_ownership_delta(root): emit(root, "vectors/topology/ownership-delta.json", "topology-ownership-delta", "ownershipDelta", "The ownership delta between two snapshots, the reorder-only case that gains and loses " - "nothing, the entry order over sixteen slots, the replica set under a shortfall, and the " - "changes that make shard identity incomparable.", + "nothing, the entry order over sixteen slots, the replica set under a shortfall, the " + "`ring` pair whose two snapshots enumerate different shard sets, and the changes that " + "make shard identity incomparable.", ["TOPO-211", "TOPO-213", "TOPO-221", "TOPO-231", "TOPO-241", "PLACE-031", "SLOT-031", - "REPL-017", "REPL-020", "SPREAD-014", "SEC-013", "ERR-010"], cases) + "RING-031", "REPL-017", "REPL-020", "SPREAD-014", "SEC-013", "ERR-010"], cases) unsupported = [{ "name": "rendezvous-enumerates-no-shard", @@ -579,6 +781,7 @@ def main(): build_defaults(root) build_identity_comparison(root) build_ownership_delta(root) + build_lineage(root) build_ring_and_pin_cases(root) print("wrote %d further vector files" % len(ENTRIES)) diff --git a/conformance/generator/generate_scenarios.py b/conformance/generator/generate_scenarios.py index 913cafa..5f60c00 100644 --- a/conformance/generator/generate_scenarios.py +++ b/conformance/generator/generate_scenarios.py @@ -37,6 +37,7 @@ # absent from this table. LEVEL_OF_SCENARIO = { "topology-rollback": "core", + "topology-acceptance-floor": "core", "caller-three-epochs-stale": "fencing", "split-topology-view": "fencing", "redirect-walk-depth-limit": "fencing", @@ -57,6 +58,9 @@ "ejection-ceiling": "failover", "probation-ramp": "failover", "outlier-comparison-set": "failover", + "handoff-local-division": "migration", + "handoff-local-fold": "migration", + "handoff-local-step-aborted": "migration", "health-reset-on-reentry": "failover", } @@ -278,6 +282,56 @@ def build_rollback_scenario(): ["TOPO-051", "TOPO-061", "TOPO-081", "TOPO-091", "ERR-031", "ERR-032"], steps) +def build_acceptance_floor_scenario(): + """`TOPO-071` and the first row of `TOPO-061`: the floor and the configured identifier. + + Both are checked before the row that installs a first document, so both hold with nothing in + force, which is the state a process is in after a restart. The rest of the acceptance table + is covered by `topology-rollback`, which runs with neither configured. + """ + from sharder_ref.topology import accept as accept_document + + setup = {"loader": {"expectedTopologyId": "migration", "minEpoch": 3}} + steps = [] + for name, document, note in [ + ("migration-epoch-2", EPOCH2, + "`TOPO-071`: the floor is below the first document, and nothing is in force, so the " + "row that installs a first document is never reached"), + ("migration-epoch-3", EPOCH3, None), + ]: + outcome, condition = accept_document(Snapshot(document), None, min_epoch=3, + expected_topology_id="migration") + step = {"action": "installTopology", + "topology": "topologies/%s.topology.json" % name, + "expect": {"outcome": outcome, "condition": condition, + "digest": jcs_digest(document)}} + if note: + step["note"] = note + if outcome == "installed": + step["expect"]["epochInForce"] = document["epoch"] + steps.insert(len(steps) if name != "migration-epoch-3" else len(steps), step) + + # The identifier row precedes the floor row, so a foreign document below the floor is a + # conflict rather than stale. Ordering is the whole point of this step. + foreign = copy.deepcopy(EPOCH1) + foreign["topologyId"] = "some-other-cluster" + outcome, condition = accept_document(Snapshot(foreign), None, min_epoch=3, + expected_topology_id="migration") + steps.insert(1, { + "action": "installTopology", "document": foreign, + "note": "`TOPO-061`: the identifier row precedes the floor row, so a foreign document " + "below `minEpoch` is a conflict and not stale", + "expect": {"outcome": outcome, "condition": condition, "digest": jcs_digest(foreign)}, + }) + + register("topology-acceptance-floor", + "A process configured with an identifier and an epoch floor, with nothing in force. " + "The floor refuses a document below it, the configured identifier refuses a foreign " + "one before the floor is reached, and the first document at or above the floor " + "installs.", + ["TOPO-061", "TOPO-071", "ERR-031", "ERR-032"], steps, setup=setup) + + def build_stale_caller_scenario(): """A caller three epochs behind, at a recipient holding the newest snapshot.""" in_force = SNAP["migration-epoch-4"] @@ -538,7 +592,8 @@ def plan_setup(plan): "toEpoch": plan.initial_target_epoch, "policy": dict(plan.policy), "handoffs": [ - {"id": handoff.id, "shard": handoff.shard, + {"id": handoff.id, "shard": handoff.shard, "sourceShard": handoff.source_shard, + "kind": handoff.kind, "source": handoff.source, "destination": handoff.destination} for handoff in plan.handoffs.values() ], @@ -613,6 +668,49 @@ def build_happy_path_scenario(): "MOVE-331", "MOVE-332", "MOVE-333"], steps) +def build_local_step_scenarios(): + """`LIN-051` through `LIN-058`: a division and a fold that happen on one node.""" + for name, kind, trigger, description in [ + ("handoff-local-division", "divide", "divideSuccess", + "A node that holds the parent under the earlier snapshot and the child under the later " + "one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to " + "`dividing` to `complete` rather than the sequence of `MOVE-021`."), + ("handoff-local-fold", "combine", "combineSuccess", + "The inverse: a node folds the copies it holds into one that matches the extent it now " + "owns, which is the step a merge ends with under `LIN-057`."), + ]: + plan = Plan(1, 2, "migration", [Handoff("l-1", "1", "n2", "n2", kind=kind, + source_shard="2")]) + steps = [] + for index, step_trigger in enumerate(["admittedByRatePolicyLocal", trigger]): + at = 1000 + index * 1000 + outcome = plan.step("l-1", step_trigger, at=at) + steps.append({"action": "handoffStep", "handoff": "l-1", "trigger": step_trigger, + "at": at, + "expect": {"outcome": outcome, "state": plan.state("l-1")}}) + steps.append({"action": "expectSummary", "expect": plan.summary()}) + register(name, description, + ["LIN-051", "LIN-052", "LIN-057", "MOVE-001", "MOVE-021", "MOVE-031"], steps, + setup=plan_setup(plan)) + + # `LIN-056`: an aborted local step is undone by the inverse hook, so `dividing` reaches + # `aborting` like every other state that does and `MOVE-233` keeps its unconditional form. + plan = Plan(1, 2, "migration", [Handoff("l-1", "1", "n2", "n2", kind="divide", + source_shard="2")]) + steps = [] + for index, step_trigger in enumerate(["admittedByRatePolicyLocal", "abort", + "rollbackSuccess"]): + at = 1000 + index * 1000 + outcome = plan.step("l-1", step_trigger, at=at) + steps.append({"action": "handoffStep", "handoff": "l-1", "trigger": step_trigger, + "at": at, "expect": {"outcome": outcome, "state": plan.state("l-1")}}) + steps.append({"action": "expectSummary", "expect": plan.summary()}) + register("handoff-local-step-aborted", + "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local " + "step is reversible and leaves the node holding the extent it held before.", + ["LIN-056", "MOVE-021", "MOVE-421"], steps, setup=plan_setup(plan)) + + def build_quiesce_lease_scenario(): """`MOVE-331` through `MOVE-336`: the commit horizon, and a lease too short to carry one.""" steps = [] @@ -1777,6 +1875,8 @@ def main(): build_split_view_scenario() build_redirect_scenario() build_happy_path_scenario() + build_local_step_scenarios() + build_acceptance_floor_scenario() build_quiesce_lease_scenario() build_node_dies_scenario() build_abort_during_catchup_scenario() diff --git a/conformance/generator/property_definitions.py b/conformance/generator/property_definitions.py index d3ae635..1c3bfbf 100644 --- a/conformance/generator/property_definitions.py +++ b/conformance/generator/property_definitions.py @@ -447,14 +447,14 @@ "requirements": ["MOVE-011", "MOVE-021", "MOVE-031", "MOVE-441", "MOVE-491"], "level": "migration", "statement": "complete, aborted, and failed admit no transition out. A handoff in " - "failed carries exactly one of the four failure kinds. An abort is " + "failed carries exactly one of the failure kinds. An abort is " "idempotent and is not admitted at or beyond verifying.", "quantifier": "for every terminal state and every trigger", "sample": {"generator": "scenario", "count": "every trigger from every terminal state"}, "check": {"form": "invariant", "statement": "a transition out of a terminal state is refused, and a handoff " "in failed carries one kind from { unverified, residue, " - "undetermined, rollbackFailed }"}, + "undetermined, rollbackFailed, undivided }"}, "witness": "scenarios/handoff-failure-kinds.json", }, { diff --git a/conformance/generator/sharder_ref/handoff.py b/conformance/generator/sharder_ref/handoff.py index 4d10b89..c7246dd 100644 --- a/conformance/generator/sharder_ref/handoff.py +++ b/conformance/generator/sharder_ref/handoff.py @@ -14,12 +14,21 @@ TERMINAL = ("complete", "aborted", "failed") -FAILURE_KINDS = ("unverified", "residue", "undetermined", "rollbackFailed") +FAILURE_KINDS = ("unverified", "residue", "undetermined", "rollbackFailed", + "undivided") # `MOVE-021`, exactly. (from, trigger) -> to TRANSITIONS = { ("planned", "admittedByRatePolicy"): "preparing", ("planned", "abort"): "aborted", + # `LIN-051`: a local step moves nothing between nodes, so it runs the short sequence. The + # admission trigger is the same one; which state it reaches is the handoff's kind. + ("planned", "admittedByRatePolicyLocal"): "dividing", + ("dividing", "divideSuccess"): "complete", + ("dividing", "combineSuccess"): "complete", + ("dividing", "abort"): "aborting", + ("dividing", "attemptsExhausted"): "aborting", + ("dividing", "dividePermanent"): "failed", ("preparing", "prepareSuccess"): "transferring", ("preparing", "abort"): "aborting", ("preparing", "attemptsExhausted"): "aborting", @@ -99,6 +108,8 @@ def retry_backoff(attempt): ("cutover", "commitUndetermined"): "undetermined", ("cleanup", "attemptsExhausted"): "residue", ("aborting", "attemptsExhausted"): "rollbackFailed", + # `MOVE-011`: a copy that matches neither the parent's extent nor the child's. + ("dividing", "dividePermanent"): "undivided", } @@ -107,9 +118,14 @@ class HandoffError(Exception): class Handoff: - def __init__(self, handoff_id, shard, source, destination, target_epoch=None): + def __init__(self, handoff_id, shard, source, destination, target_epoch=None, + kind="handoff", source_shard=None): self.id = handoff_id self.shard = shard + self.kind = kind + # `LIN-041`: the shard the contents come from, which differs from `shard` only where a + # lineage divided or folded an extent. + self.source_shard = source_shard or shard self.source = source self.destination = destination self.state = "planned" @@ -479,6 +495,20 @@ def rebase(self, topology_id, epoch, replica_sets, at=None, comparable=True): "rebaseInterval": self.rebase_interval()} +def replica_set(snapshot, shard, factor_of): + """`TOPO-211`: the entries of a shard's preference list whose role is `replica`. + + The replica count is the achieved count `r` of `REPL-020` rather than the configured factor, so + a shortfall shortens the prefix and the entries beyond it are the fallback tail of `REPL-013`. + Both the ownership delta and the lineage read a replica set through this one definition. + """ + from . import placement, routing as route_module + + ordering = placement.candidates_for_shard(snapshot, shard, snapshot.placement_set) + entries, r, _, _ = route_module.build_preference_list(snapshot, ordering, factor_of(snapshot)) + return [e["node"] for e in entries[:r]] + + def ownership_delta(before, after, factor_of): """`TOPO-211`: the shards whose ordered replica set differs, with nodes gained and lost. @@ -498,16 +528,8 @@ def ownership_delta(before, after, factor_of): ordered = order_after + [s for s in order_before if s not in shards_after] rows = [] for shard in ordered: - def replicas(snapshot, present): - if not present: - return [] - ordering = placement.candidates_for_shard(snapshot, shard, snapshot.placement_set) - entries, r, _, _ = route_module.build_preference_list( - snapshot, ordering, factor_of(snapshot)) - return [e["node"] for e in entries[:r]] - - was = replicas(before, shard in shards_before) - now = replicas(after, shard in shards_after) + was = replica_set(before, shard, factor_of) if shard in shards_before else [] + now = replica_set(after, shard, factor_of) if shard in shards_after else [] if was != now: rows.append({ "shard": shard, diff --git a/conformance/generator/sharder_ref/lineage.py b/conformance/generator/sharder_ref/lineage.py new file mode 100644 index 0000000..f6b85b4 --- /dev/null +++ b/conformance/generator/sharder_ref/lineage.py @@ -0,0 +1,321 @@ +"""Shard lineage: the correspondence between the extents of two snapshots. + +`LIN-001` through `LIN-045`. A lineage relates extents rather than shard identifiers, so it +answers where a shard's contents come from when an epoch changes which shards exist. An ownership +delta joins two snapshots on the identifier and has no answer there. + +The extent representation is per strategy, under `LIN-011` through `LIN-015`. Under `ring` an +extent is a set of half-open segments of the 64-bit hash space, which wraps at zero, so containment +and union are interval arithmetic. Under `slot` `TOPO-231` holds `slotCount` equal, so every extent +equals its counterpart and the lineage is the identity of `LIN-007`. Under `directory` this module +refuses a pair whose shard sets differ, which is the refusal `LIN-013` states. +""" + +from . import placement + +SPACE = 1 << 64 + + +class Refused(Exception): + """`ERR-050`: a plan refused with the cause the lineage names.""" + + def __init__(self, cause, detail=None): + super().__init__(cause if detail is None else "%s: %s" % (cause, detail)) + self.cause = cause + self.detail = detail + + +# ------------------------------------------------------------------ interval extents under `ring` + +def _segments(low_exclusive, high_inclusive): + """The keys `h` with `low_exclusive < h <= high_inclusive`, as half-open `[start, end)` pairs. + + `RING-020` gives the owning entry that interval, and it wraps at zero, so a wrapping interval is + two segments rather than one. + """ + start, end = (low_exclusive + 1) % SPACE, (high_inclusive + 1) % SPACE + if start == end: + return ((0, SPACE),) + if start < end: + return ((start, end),) + return ((0, end), (start, SPACE)) + + +def ring_extents(snapshot): + """`LIN-011`: each distinct token's interval, in the ascending ring order of `RING-031`.""" + tokens = [int(s, 16) for s in placement.ring_shards(snapshot)] + out = {} + for position, token in enumerate(tokens): + predecessor = tokens[position - 1] if len(tokens) > 1 else token + out[placement.hashing.hex_u64(token)] = _segments(predecessor, token) + return out + + +def _measure(segments): + return sum(end - start for start, end in segments) + + +def _intersection(first, second): + out = [] + for a_start, a_end in first: + for b_start, b_end in second: + start, end = max(a_start, b_start), min(a_end, b_end) + if start < end: + out.append((start, end)) + return tuple(sorted(out)) + + +def meets(first, second): + return bool(_intersection(first, second)) + + +def contains(outer, inner): + return _measure(_intersection(outer, inner)) == _measure(inner) + + +def equal(first, second): + return _measure(first) == _measure(second) == _measure(_intersection(first, second)) + + +# ------------------------------------------------------- extents under `directory` + +def _matchers(snapshot): + """Each entry's matcher as `(kind, decoded octets)`, in `entries` array order.""" + from .topology import decode_matcher_value + + out = [] + for entry in snapshot.strategy["entries"]: + match = entry["match"] + out.append((match["kind"], decode_matcher_value(match))) + return out + + +def _regions(first, second): + """The regions the two tables' matcher values cut the keyspace into. + + Every node of the combined prefix trie contributes two regions: the key equal to that node, and + the keys strictly extending it that no deeper node is a prefix of. The root contributes the + second of those for the keys no matcher value is a prefix of. Two keys in one region match + exactly the same matchers of either table, so a region is the finest distinction either table + can draw and the regions together cover the keyspace. + """ + nodes = {b""} + for kind, value in list(first) + list(second): + nodes.add(value) + ordered = sorted(nodes) + return ([("exact", node) for node in ordered if node] + + [("open", node) for node in ordered]) + + +def _winner(matchers, region): + """`PLACE-065`: the entry index that wins a region, or `None` where no entry matches it. + + An `exact` matcher can only win the region that is its own key, because a region of keys + strictly extending a node contains no node of the trie and every matcher value is one. + """ + shape, node = region + if shape == "exact": + for index, (kind, value) in enumerate(matchers): + if kind == "exact" and value == node: + return index + best = None + for index, (kind, value) in enumerate(matchers): + if kind != "prefix" or not node.startswith(value): + continue + if best is None or len(value) > len(matchers[best][1]): + best = index + return best + + +def directory_extents(before, after): + """`LIN-013`: each shard's extent as the set of regions its entry wins.""" + first, second = _matchers(before), _matchers(after) + regions = _regions(first, second) + out = [] + for snapshot, matchers in ((before, first), (after, second)): + shards = placement.shards(snapshot) + extents = {shard: set() for shard in shards} + for region in regions: + index = _winner(matchers, region) + if index is not None: + extents[shards[index]].add(region) + out.append({shard: frozenset(region) for shard, region in extents.items()}) + return out[0], out[1] + + +def _region_meets(first, second): + return bool(first & second) + + +def _region_contains(outer, inner): + return inner <= outer + + +# ------------------------------------------------------------------------------- the lineage + +def extents(snapshot): + """The extent of each shard the snapshot enumerates, under `LIN-011` through `LIN-014`.""" + kind = snapshot.strategy["kind"] + if kind == "ring": + return ring_extents(snapshot) + if kind == "rendezvous": + raise Refused("strategyUnsupported", "rendezvous enumerates no shard") + # `LIN-012` and `LIN-013`: an identity extent, keyed by the shard identifier itself. Under + # `slot` `TOPO-231` holds `slotCount` equal; under `directory` a differing shard set is refused + # by `classify` before an extent is compared. + return {shard: shard for shard in placement.shards(snapshot)} + + +def _identity_lineage(before, after, replicas_of): + rows = [] + for shard in placement.shards(after): + was, now = replicas_of(before, shard), replicas_of(after, shard) + rows.append({"shard": shard, "class": "unchanged" if was == now else "moved", + "parents": [shard]}) + return rows + + +def comparable(before, after): + """`TOPO-231`: the member the two snapshots differ in, or `None` where they are comparable. + + Shard identity is the join both an ownership delta and a lineage rest on, so the two joins + share one comparability rule and `LIN-006` refuses exactly where `TOPO-231` does. + """ + if before.strategy["kind"] != after.strategy["kind"]: + return "strategy.kind" + if before.key_transform != after.key_transform: + return "keyTransform" + if before.seed != after.seed: + return "hash.seed" + if before.hash_config["algorithm"] != after.hash_config["algorithm"]: + return "hash.algorithm" + if before.strategy["kind"] == "slot" \ + and before.strategy["slotCount"] != after.strategy["slotCount"]: + return "slotCount" + return None + + +def classify(before, after, replicas_of): + """`LIN-021`: each shard of the later snapshot, then each shard only the earlier one holds. + + `LIN-033` fixes that order, which is the order `TOPO-213` fixes for an ownership delta. + """ + differing = comparable(before, after) + if differing is not None: + raise Refused("incomparableShards", differing) + kind = before.strategy["kind"] + if kind == "rendezvous": + raise Refused("strategyUnsupported", "rendezvous enumerates no shard") + + shards_before, shards_after = placement.shards(before), placement.shards(after) + if kind == "slot": + # `LIN-012`: `TOPO-231` has already refused a differing `slotCount`, so the two snapshots + # enumerate the same slots and every extent equals its counterpart. + return _identity_lineage(before, after, replicas_of) + + # `LIN-007`: an equal shard set is an equal extent set, so the lineage is the identity. + if shards_before == shards_after: + return _identity_lineage(before, after, replicas_of) + + earlier, later, meets_, contains_, equal_ = _geometry(before, after, kind) + rows = [] + for shard in shards_after: + mine = later[shard] + parents = [s for s in shards_before if meets_(earlier[s], mine)] + for parent in parents: + if not contains_(earlier[parent], mine) and not contains_(mine, earlier[parent]): + raise Refused("unalignedLineage", shard) + if not parents: + rows.append({"shard": shard, "class": "fresh", "parents": []}) + elif len(parents) > 1: + rows.append({"shard": shard, "class": "merged", "parents": parents}) + elif equal_(earlier[parents[0]], mine): + was, now = replicas_of(before, shard), replicas_of(after, shard) + rows.append({"shard": shard, "class": "unchanged" if was == now else "moved", + "parents": parents}) + else: + rows.append({"shard": shard, "class": "divided", "parents": parents}) + + for shard in shards_before: + if shard in later: + continue + mine = earlier[shard] + children = [s for s in shards_after if meets_(later[s], mine)] + if not children: + rows.append({"shard": shard, "class": "vacated", "parents": []}) + elif len(children) > 1: + rows.append({"shard": shard, "class": "split", "parents": children}) + else: + rows.append({"shard": shard, "class": "folded", "parents": children}) + return rows + + +def _geometry(before, after, kind): + """The two extent maps and the three predicates that decide them, per strategy kind.""" + if kind == "directory": + earlier, later = directory_extents(before, after) + return earlier, later, _region_meets, _region_contains, lambda a, b: a == b + return ring_extents(before), ring_extents(after), meets, contains, equal + + +def parents_of(before, after): + """For each shard of the later snapshot, the shards of the earlier one its extent draws from.""" + kind = before.strategy["kind"] + shards_before, shards_after = placement.shards(before), placement.shards(after) + if kind == "slot" or shards_before == shards_after: + return {shard: [shard] for shard in shards_after} + earlier, later, meets_, _, _ = _geometry(before, after, kind) + return {shard: [s for s in shards_before if meets_(earlier[s], later[shard])] + for shard in shards_after} + + +def plan_handoffs(before, after, replicas_of): + """`LIN-041` through `LIN-058`: the handoffs of a plan and the local steps beside them. + + `LIN-057` fixes the order within one shard: a division is sequenced before every handoff that + draws from the divided parent, and a fold after every handoff that draws into the folded shard. + """ + parents = parents_of(before, after) + kind = before.strategy["kind"] + identity = kind == "slot" or placement.shards(before) == placement.shards(after) + if identity: + earlier, later, equal_ = {}, {}, None + else: + earlier, later, _, _, equal_ = _geometry(before, after, kind) + + out = [] + for shard in placement.shards(after): + destinations = replicas_of(after, shard) + mine = parents[shard] + divisions, folds, moves = [], [], [] + if not identity and len(mine) == 1 and not equal_(earlier[mine[0]], later[shard]): + # `LIN-052`: the extent narrowed, so a node holding both divides its own copy. + for node in destinations: + if node in replicas_of(before, mine[0]): + divisions.append({"shard": shard, "sourceShard": mine[0], "kind": "divide", + "source": node, "destination": node}) + if not identity and len(mine) > 1: + # `LIN-052`: the extent widened, so a node holding a parent folds what it holds. + for node in destinations: + if any(node in replicas_of(before, parent) for parent in mine): + folds.append({"shard": shard, "sourceShard": shard, "kind": "combine", + "source": node, "destination": node}) + for parent in mine: + held = replicas_of(before, parent) + if not held: + continue + needing = [d for d in destinations if d not in held] + departing = [n for n in held if n not in destinations] + for position, destination in enumerate(needing): + # `LIN-045`: a replica giving the shard up is drained before one keeping it. + source = departing[position] if position < len(departing) else held[0] + moves.append({"shard": shard, "sourceShard": parent, "kind": "handoff", + "source": source, "destination": destination}) + out.extend(divisions + moves + folds) + + for position, entry in enumerate(out): + same = [e for e in out[:position] if e["shard"] == entry["shard"]] + prefix = "l-" if entry["kind"] != "handoff" else "h-" + entry["id"] = "%s%s" % (prefix, entry["shard"]) if not same \ + else "%s%s-%d" % (prefix, entry["shard"], len(same)) + return out diff --git a/conformance/manifest.json b/conformance/manifest.json index 1080f87..84f38bd 100644 --- a/conformance/manifest.json +++ b/conformance/manifest.json @@ -1,9 +1,9 @@ { "suite": "sharder conformance suite", "revision": { - "id": "e91417161ae0de0002cf3748a9ecd969e82c6233f0d8ab97fd3e908492b24204", + "id": "c29e6a8f3facafa56414e46851ada1610db5e64c016714a8dba915e08875d57c", "basis": "sha256 of the RFC 8785 canonical form of the file digests, the level table, and the strategy surfaces", - "fileCount": 193, + "fileCount": 206, "trees": [ "vectors", "topologies", @@ -15,13 +15,13 @@ "design": "docs/design/30-conformance.md", "generator": "conformance/generator", "counts": { - "vectorFiles": 67, - "vectorCases": 596, - "topologyDocuments": 102, - "scenarios": 22, - "scenarioSteps": 303, + "vectorFiles": 69, + "vectorCases": 609, + "topologyDocuments": 109, + "scenarios": 26, + "scenarioSteps": 316, "properties": 28, - "requirementsNamed": 454 + "requirementsNamed": 474 }, "requirementsNamed": [ "CFG-014", @@ -189,6 +189,26 @@ "KEY-042", "KEY-043", "KEY-044", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-016", + "LIN-021", + "LIN-022", + "LIN-031", + "LIN-033", + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-056", + "LIN-057", "MOVE-001", "MOVE-011", "MOVE-021", @@ -509,10 +529,10 @@ ], "surface": "the document pipeline, the snapshot lifecycle, and what is reported", "vectorFiles": 8, - "vectorCases": 133, - "scenarios": 1, + "vectorCases": 135, + "scenarios": 2, "properties": 22, - "requirementsNamed": 125 + "requirementsNamed": 126 }, { "level": "scale", @@ -568,11 +588,11 @@ "failover" ], "surface": "the handoff coordinator and rate control", - "vectorFiles": 3, - "vectorCases": 15, - "scenarios": 12, + "vectorFiles": 5, + "vectorCases": 26, + "scenarios": 15, "properties": 3, - "requirementsNamed": 80 + "requirementsNamed": 103 } ], "strategySurfaces": [ @@ -663,6 +683,38 @@ "digest": "ffdb65dd9fb5976bf5dcb760056c229d37619bd2788c6f3d490ef37bd4e8d5eb", "valid": true }, + "topologies/delta-ring-added.topology.json": { + "topologyId": "delta-ring", + "epoch": 2, + "strategy": "ring", + "nodeCount": 3, + "digest": "2042c8625badd5ccb5019d181246a66b1b017074612f589c7a3f2c8dfe2aa142", + "valid": true + }, + "topologies/delta-ring-before.topology.json": { + "topologyId": "delta-ring", + "epoch": 1, + "strategy": "ring", + "nodeCount": 2, + "digest": "425a7afcc4eb05c3403cbde0da416622e2785d02c55a590a63be99a71c90dd46", + "valid": true + }, + "topologies/delta-ring-removed.topology.json": { + "topologyId": "delta-ring", + "epoch": 3, + "strategy": "ring", + "nodeCount": 2, + "digest": "106ba6977b64947076612f7cfa874adfb2b68c2fccedcd64bfce2eab612371ab", + "valid": true + }, + "topologies/delta-ring-unaligned.topology.json": { + "topologyId": "delta-ring", + "epoch": 4, + "strategy": "ring", + "nodeCount": 3, + "digest": "9856ceda5c5b9ca5bdd4240396b79dffc1fdb51d8c0d542017fe0a70088df6f2", + "valid": true + }, "topologies/delta-short-after.topology.json": { "topologyId": "delta-short", "epoch": 2, @@ -1007,6 +1059,30 @@ "digest": "7797a43770f84df98db19f9d11329d55243c779d050fd21cce026f985e0394bb", "valid": true }, + "topologies/lineage-directory-before.topology.json": { + "topologyId": "lineage-directory", + "epoch": 1, + "strategy": "directory", + "nodeCount": 2, + "digest": "202c08ccc95f89693fff2411d987ea83bf6e54536b0ff0434d7b77027cf2588d", + "valid": true + }, + "topologies/lineage-directory-fresh.topology.json": { + "topologyId": "lineage-directory", + "epoch": 3, + "strategy": "directory", + "nodeCount": 2, + "digest": "3ef0ed21f04db339d03aac9c12d7d6693578cb72df336c453716e77030559697", + "valid": true + }, + "topologies/lineage-directory-refined.topology.json": { + "topologyId": "lineage-directory", + "epoch": 2, + "strategy": "directory", + "nodeCount": 2, + "digest": "88b5a9b7766b11d4cee599ccff278e02f6321850547b5012273d59c1a8f2899b", + "valid": true + }, "topologies/migration-epoch-1.topology.json": { "topologyId": "migration", "epoch": 1, @@ -1682,7 +1758,7 @@ "ERR-021", "REPL-025" ], - "sha256": "0c944ea2e167f309384f731a76f7d3ff8407094bb6d7fdab7f1401276ae2cd8a" + "sha256": "8e923beb7e6e5d839a6df17d214722e7ce37f960b70fb0f21e0e9591b9a5dcec" }, { "file": "vectors/formulas/attempt-limit.json", @@ -1976,6 +2052,67 @@ ], "sha256": "ae9cde46271a4ee1650e11cab0e94da757ea92c83b5eb5e447a50fff6e7679fe" }, + { + "file": "vectors/migration/lineage.json", + "vectorSet": "migration-lineage", + "kind": "lineage", + "level": "migration", + "strategies": [], + "caseStrategies": [ + "directory", + "rendezvous", + "ring", + "slot" + ], + "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` prefix refined and the fresh extent beside it, the incomparable pair, and the kind that enumerates no shard.", + "topology": null, + "caseCount": 8, + "requirements": [ + "DIR-002", + "DIR-010", + "ERR-050", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-016", + "LIN-021", + "LIN-022", + "LIN-031", + "LIN-033", + "MOVE-241", + "PLACE-065", + "TOPO-231" + ], + "sha256": "9a705c589dc85a46f81d0d3c9eee12883b170fe29ec8edb3088849a3ad273138" + }, + { + "file": "vectors/migration/plan-construction.json", + "vectorSet": "migration-plan-construction", + "kind": "planConstruction", + "level": "migration", + "strategies": [], + "caseStrategies": [ + "ring", + "slot" + ], + "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, the local steps beside them and their order, and the plan under the identity lineage.", + "topology": null, + "caseCount": 3, + "requirements": [ + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" + ], + "sha256": "97e1e1999b409306aa3ec272c8aad95aaeef61164012416a2e82f8eec034afbf" + }, { "file": "vectors/movement/add-one-node.json", "vectorSet": "movement-add-one-node", @@ -2054,7 +2191,7 @@ "OBS-020", "OBS-021" ], - "sha256": "548ce6104a96251eea8bdd586c79069f5d0bdbb9968b16426a5f170187cce189" + "sha256": "cea1227356a1ea21fd617f8a149d503ffd226757e8f7eeb34b4a904d7e1bea70" }, { "file": "vectors/observability/inventory-routing.json", @@ -2981,16 +3118,18 @@ "level": "core", "strategies": [], "caseStrategies": [ + "ring", "slot" ], - "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, and the changes that make shard identity incomparable.", + "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, the `ring` pair whose two snapshots enumerate different shard sets, and the changes that make shard identity incomparable.", "topology": null, - "caseCount": 6, + "caseCount": 8, "requirements": [ "ERR-010", "PLACE-031", "REPL-017", "REPL-020", + "RING-031", "SEC-013", "SLOT-031", "SPREAD-014", @@ -3000,7 +3139,7 @@ "TOPO-231", "TOPO-241" ], - "sha256": "df96523b00ab2b90758bb38693770b4e54155dd040bf7e1b366e206c2ede68aa" + "sha256": "1d5d287fe69b73ca6954c3640081356ba38c67e768d82eff3cd1ff54b4a7f618" }, { "file": "vectors/validation/documents.json", @@ -3140,6 +3279,61 @@ "TOPO-221" ] }, + { + "file": "scenarios/handoff-local-division.json", + "scenario": "handoff-local-division", + "level": "migration", + "description": "A node that holds the parent under the earlier snapshot and the child under the later one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to `dividing` to `complete` rather than the sequence of `MOVE-021`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-fold.json", + "scenario": "handoff-local-fold", + "level": "migration", + "description": "The inverse: a node folds the copies it holds into one that matches the extent it now owns, which is the step a merge ends with under `LIN-057`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-step-aborted.json", + "scenario": "handoff-local-step-aborted", + "level": "migration", + "description": "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local step is reversible and leaves the node holding the extent it held before.", + "stepCount": 4, + "requirements": [ + "LIN-056", + "MOVE-021", + "MOVE-421" + ] + }, + { + "file": "scenarios/topology-acceptance-floor.json", + "scenario": "topology-acceptance-floor", + "level": "core", + "description": "A process configured with an identifier and an epoch floor, with nothing in force. The floor refuses a document below it, the configured identifier refuses a foreign one before the floor is reached, and the first document at or above the floor installs.", + "stepCount": 3, + "requirements": [ + "ERR-031", + "ERR-032", + "TOPO-061", + "TOPO-071" + ] + }, { "file": "scenarios/quiesce-lease-expiry.json", "scenario": "quiesce-lease-expiry", diff --git a/conformance/properties/properties.json b/conformance/properties/properties.json index a5fbc3c..9a7d4c1 100644 --- a/conformance/properties/properties.json +++ b/conformance/properties/properties.json @@ -794,7 +794,7 @@ "MOVE-491" ], "level": "migration", - "statement": "complete, aborted, and failed admit no transition out. A handoff in failed carries exactly one of the four failure kinds. An abort is idempotent and is not admitted at or beyond verifying.", + "statement": "complete, aborted, and failed admit no transition out. A handoff in failed carries exactly one of the failure kinds. An abort is idempotent and is not admitted at or beyond verifying.", "quantifier": "for every terminal state and every trigger", "sample": { "generator": "scenario", @@ -802,7 +802,7 @@ }, "check": { "form": "invariant", - "statement": "a transition out of a terminal state is refused, and a handoff in failed carries one kind from { unverified, residue, undetermined, rollbackFailed }" + "statement": "a transition out of a terminal state is refused, and a handoff in failed carries one kind from { unverified, residue, undetermined, rollbackFailed, undivided }" }, "witness": "scenarios/handoff-failure-kinds.json" }, diff --git a/conformance/scenarios/abort-during-catching-up.json b/conformance/scenarios/abort-during-catching-up.json index fd6af0e..806ce35 100644 --- a/conformance/scenarios/abort-during-catching-up.json +++ b/conformance/scenarios/abort-during-catching-up.json @@ -25,6 +25,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/coordinator-death-and-recovery.json b/conformance/scenarios/coordinator-death-and-recovery.json index 0b0e14d..355905b 100644 --- a/conformance/scenarios/coordinator-death-and-recovery.json +++ b/conformance/scenarios/coordinator-death-and-recovery.json @@ -32,6 +32,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/handoff-failure-kinds.json b/conformance/scenarios/handoff-failure-kinds.json index c0aa3c0..e2981a5 100644 --- a/conformance/scenarios/handoff-failure-kinds.json +++ b/conformance/scenarios/handoff-failure-kinds.json @@ -30,6 +30,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/handoff-local-division.json b/conformance/scenarios/handoff-local-division.json new file mode 100644 index 0000000..efc1aea --- /dev/null +++ b/conformance/scenarios/handoff-local-division.json @@ -0,0 +1,77 @@ +{ + "scenario": "handoff-local-division", + "level": "migration", + "description": "A node that holds the parent under the earlier snapshot and the child under the later one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to `dividing` to `complete` rather than the sequence of `MOVE-021`.", + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ], + "deterministic": true, + "setup": { + "plan": { + "topologyId": "migration", + "fromEpoch": 1, + "toEpoch": 2, + "policy": { + "maxConcurrentHandoffs": 4, + "maxConcurrentPerSourceNode": 1, + "maxConcurrentPerDestinationNode": 1, + "initialStepBudget": 1, + "commitDeadlineMillis": 30000, + "quiesceLeaseMarginMillis": 1000 + }, + "handoffs": [ + { + "id": "l-1", + "shard": "1", + "sourceShard": "2", + "kind": "divide", + "source": "n2", + "destination": "n2" + } + ] + } + }, + "steps": [ + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "admittedByRatePolicyLocal", + "at": 1000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "planned", + "toState": "dividing" + }, + "state": "dividing" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "divideSuccess", + "at": 2000, + "expect": { + "outcome": { + "outcome": "settled", + "id": "l-1", + "terminalState": "complete", + "failureKind": null + }, + "state": "complete" + } + }, + { + "action": "expectSummary", + "expect": { + "complete": 1 + } + } + ] +} diff --git a/conformance/scenarios/handoff-local-fold.json b/conformance/scenarios/handoff-local-fold.json new file mode 100644 index 0000000..416ec65 --- /dev/null +++ b/conformance/scenarios/handoff-local-fold.json @@ -0,0 +1,77 @@ +{ + "scenario": "handoff-local-fold", + "level": "migration", + "description": "The inverse: a node folds the copies it holds into one that matches the extent it now owns, which is the step a merge ends with under `LIN-057`.", + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ], + "deterministic": true, + "setup": { + "plan": { + "topologyId": "migration", + "fromEpoch": 1, + "toEpoch": 2, + "policy": { + "maxConcurrentHandoffs": 4, + "maxConcurrentPerSourceNode": 1, + "maxConcurrentPerDestinationNode": 1, + "initialStepBudget": 1, + "commitDeadlineMillis": 30000, + "quiesceLeaseMarginMillis": 1000 + }, + "handoffs": [ + { + "id": "l-1", + "shard": "1", + "sourceShard": "2", + "kind": "combine", + "source": "n2", + "destination": "n2" + } + ] + } + }, + "steps": [ + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "admittedByRatePolicyLocal", + "at": 1000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "planned", + "toState": "dividing" + }, + "state": "dividing" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "combineSuccess", + "at": 2000, + "expect": { + "outcome": { + "outcome": "settled", + "id": "l-1", + "terminalState": "complete", + "failureKind": null + }, + "state": "complete" + } + }, + { + "action": "expectSummary", + "expect": { + "complete": 1 + } + } + ] +} diff --git a/conformance/scenarios/handoff-local-step-aborted.json b/conformance/scenarios/handoff-local-step-aborted.json new file mode 100644 index 0000000..8f71ad6 --- /dev/null +++ b/conformance/scenarios/handoff-local-step-aborted.json @@ -0,0 +1,89 @@ +{ + "scenario": "handoff-local-step-aborted", + "level": "migration", + "description": "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local step is reversible and leaves the node holding the extent it held before.", + "requirements": [ + "LIN-056", + "MOVE-021", + "MOVE-421" + ], + "deterministic": true, + "setup": { + "plan": { + "topologyId": "migration", + "fromEpoch": 1, + "toEpoch": 2, + "policy": { + "maxConcurrentHandoffs": 4, + "maxConcurrentPerSourceNode": 1, + "maxConcurrentPerDestinationNode": 1, + "initialStepBudget": 1, + "commitDeadlineMillis": 30000, + "quiesceLeaseMarginMillis": 1000 + }, + "handoffs": [ + { + "id": "l-1", + "shard": "1", + "sourceShard": "2", + "kind": "divide", + "source": "n2", + "destination": "n2" + } + ] + } + }, + "steps": [ + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "admittedByRatePolicyLocal", + "at": 1000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "planned", + "toState": "dividing" + }, + "state": "dividing" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "abort", + "at": 2000, + "expect": { + "outcome": { + "outcome": "advanced", + "id": "l-1", + "fromState": "dividing", + "toState": "aborting" + }, + "state": "aborting" + } + }, + { + "action": "handoffStep", + "handoff": "l-1", + "trigger": "rollbackSuccess", + "at": 3000, + "expect": { + "outcome": { + "outcome": "settled", + "id": "l-1", + "terminalState": "aborted", + "failureKind": null + }, + "state": "aborted" + } + }, + { + "action": "expectSummary", + "expect": { + "aborted": 1 + } + } + ] +} diff --git a/conformance/scenarios/index.json b/conformance/scenarios/index.json index f92dfcd..e9c2dd5 100644 --- a/conformance/scenarios/index.json +++ b/conformance/scenarios/index.json @@ -1,7 +1,7 @@ { "suite": "sharder simulation scenarios", - "scenarioCount": 22, - "stepCount": 303, + "scenarioCount": 26, + "stepCount": 316, "requirementsCovered": [ "CFG-030", "CFG-040", @@ -77,6 +77,10 @@ "HEALTH-052", "HEALTH-053", "HEALTH-055", + "LIN-051", + "LIN-052", + "LIN-056", + "LIN-057", "MOVE-001", "MOVE-011", "MOVE-021", @@ -138,6 +142,7 @@ "SEC-031", "TOPO-051", "TOPO-061", + "TOPO-071", "TOPO-081", "TOPO-091", "TOPO-111", @@ -259,6 +264,61 @@ "TOPO-221" ] }, + { + "file": "scenarios/handoff-local-division.json", + "scenario": "handoff-local-division", + "level": "migration", + "description": "A node that holds the parent under the earlier snapshot and the child under the later one divides its own copy. Nothing crosses the network, so the handoff runs `planned` to `dividing` to `complete` rather than the sequence of `MOVE-021`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-fold.json", + "scenario": "handoff-local-fold", + "level": "migration", + "description": "The inverse: a node folds the copies it holds into one that matches the extent it now owns, which is the step a merge ends with under `LIN-057`.", + "stepCount": 3, + "requirements": [ + "LIN-051", + "LIN-052", + "LIN-057", + "MOVE-001", + "MOVE-021", + "MOVE-031" + ] + }, + { + "file": "scenarios/handoff-local-step-aborted.json", + "scenario": "handoff-local-step-aborted", + "level": "migration", + "description": "`LIN-056`: a division that is aborted is undone by the inverse hook, so the local step is reversible and leaves the node holding the extent it held before.", + "stepCount": 4, + "requirements": [ + "LIN-056", + "MOVE-021", + "MOVE-421" + ] + }, + { + "file": "scenarios/topology-acceptance-floor.json", + "scenario": "topology-acceptance-floor", + "level": "core", + "description": "A process configured with an identifier and an epoch floor, with nothing in force. The floor refuses a document below it, the configured identifier refuses a foreign one before the floor is reached, and the first document at or above the floor installs.", + "stepCount": 3, + "requirements": [ + "ERR-031", + "ERR-032", + "TOPO-061", + "TOPO-071" + ] + }, { "file": "scenarios/quiesce-lease-expiry.json", "scenario": "quiesce-lease-expiry", diff --git a/conformance/scenarios/migration-rate-control.json b/conformance/scenarios/migration-rate-control.json index 5db3f0c..5f7e5d4 100644 --- a/conformance/scenarios/migration-rate-control.json +++ b/conformance/scenarios/migration-rate-control.json @@ -29,12 +29,16 @@ { "id": "h-a", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n4" }, { "id": "h-b", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n1", "destination": "n3" } diff --git a/conformance/scenarios/node-dies-mid-migration.json b/conformance/scenarios/node-dies-mid-migration.json index 6568fd5..1df4ad2 100644 --- a/conformance/scenarios/node-dies-mid-migration.json +++ b/conformance/scenarios/node-dies-mid-migration.json @@ -31,6 +31,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/plan-superseded-by-new-epoch.json b/conformance/scenarios/plan-superseded-by-new-epoch.json index 766c8a2..1a0347c 100644 --- a/conformance/scenarios/plan-superseded-by-new-epoch.json +++ b/conformance/scenarios/plan-superseded-by-new-epoch.json @@ -30,18 +30,24 @@ { "id": "h-a", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" }, { "id": "h-b", "shard": "2", + "sourceShard": "2", + "kind": "handoff", "source": "n3", "destination": "n1" }, { "id": "h-c", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n3" } diff --git a/conformance/scenarios/quiesce-lease-expiry.json b/conformance/scenarios/quiesce-lease-expiry.json index 0f89c87..0177867 100644 --- a/conformance/scenarios/quiesce-lease-expiry.json +++ b/conformance/scenarios/quiesce-lease-expiry.json @@ -30,6 +30,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/scenarios/rebase-drops-a-handoff.json b/conformance/scenarios/rebase-drops-a-handoff.json index 383abb8..8733afe 100644 --- a/conformance/scenarios/rebase-drops-a-handoff.json +++ b/conformance/scenarios/rebase-drops-a-handoff.json @@ -33,12 +33,16 @@ { "id": "h-0", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n5" }, { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n3", "destination": "n6" } diff --git a/conformance/scenarios/rebase-past-the-cutover.json b/conformance/scenarios/rebase-past-the-cutover.json index e7e52ba..f02c84b 100644 --- a/conformance/scenarios/rebase-past-the-cutover.json +++ b/conformance/scenarios/rebase-past-the-cutover.json @@ -29,12 +29,16 @@ { "id": "h-0", "shard": "0", + "sourceShard": "0", + "kind": "handoff", "source": "n1", "destination": "n5" }, { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n3", "destination": "n6" } diff --git a/conformance/scenarios/topology-acceptance-floor.json b/conformance/scenarios/topology-acceptance-floor.json new file mode 100644 index 0000000..2119521 --- /dev/null +++ b/conformance/scenarios/topology-acceptance-floor.json @@ -0,0 +1,133 @@ +{ + "scenario": "topology-acceptance-floor", + "level": "core", + "description": "A process configured with an identifier and an epoch floor, with nothing in force. The floor refuses a document below it, the configured identifier refuses a foreign one before the floor is reached, and the first document at or above the floor installs.", + "requirements": [ + "ERR-031", + "ERR-032", + "TOPO-061", + "TOPO-071" + ], + "deterministic": true, + "setup": { + "loader": { + "expectedTopologyId": "migration", + "minEpoch": 3 + } + }, + "steps": [ + { + "action": "installTopology", + "topology": "topologies/migration-epoch-2.topology.json", + "expect": { + "outcome": "rejected", + "condition": { + "code": 203, + "name": "staleDocument", + "detail": "an epoch below minEpoch" + }, + "digest": "88174beca86efc281ac17c76a28b20728055b2a526d2cd9a216a3ca5f2307bb8" + }, + "note": "`TOPO-071`: the floor is below the first document, and nothing is in force, so the row that installs a first document is never reached" + }, + { + "action": "installTopology", + "document": { + "formatVersion": "1.0", + "topologyId": "some-other-cluster", + "epoch": 1, + "domainLevels": [ + "zone" + ], + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "slot", + "slotCount": 3, + "assignment": "explicit", + "assignments": [ + { + "slots": [ + "0" + ], + "nodes": [ + "n1", + "n2" + ] + }, + { + "slots": [ + "1" + ], + "nodes": [ + "n2", + "n3" + ] + }, + { + "slots": [ + "2" + ], + "nodes": [ + "n3", + "n1" + ] + } + ] + }, + "nodes": [ + { + "id": "n1", + "domains": { + "zone": "za" + }, + "address": "10.0.0.1:7000" + }, + { + "id": "n2", + "domains": { + "zone": "zb" + }, + "address": "10.0.0.2:7000" + }, + { + "id": "n3", + "domains": { + "zone": "zc" + }, + "address": "10.0.0.3:7000" + }, + { + "id": "n4", + "state": "joining", + "domains": { + "zone": "zd" + }, + "address": "10.0.0.4:7000" + } + ] + }, + "note": "`TOPO-061`: the identifier row precedes the floor row, so a foreign document below `minEpoch` is a conflict and not stale", + "expect": { + "outcome": "rejected", + "condition": { + "code": 202, + "name": "topologyConflict", + "detail": "a differing topologyId" + }, + "digest": "5dc597d8f3c39ade58e5f7fcb16804f6ce3351d8da5859c295491b439f40343f" + } + }, + { + "action": "installTopology", + "topology": "topologies/migration-epoch-3.topology.json", + "expect": { + "outcome": "installed", + "condition": null, + "digest": "135af637f87fa8f69f32fe0be4f150501c505d8ce4c49db464c4c15856baa99c", + "epochInForce": 3 + } + } + ] +} diff --git a/conformance/scenarios/undetermined-resolves-both-ways.json b/conformance/scenarios/undetermined-resolves-both-ways.json index c684cf2..518e27d 100644 --- a/conformance/scenarios/undetermined-resolves-both-ways.json +++ b/conformance/scenarios/undetermined-resolves-both-ways.json @@ -37,6 +37,8 @@ { "id": "h-1", "shard": "1", + "sourceShard": "1", + "kind": "handoff", "source": "n2", "destination": "n4" } diff --git a/conformance/topologies/delta-ring-added.topology.json b/conformance/topologies/delta-ring-added.topology.json new file mode 100644 index 0000000..dc80006 --- /dev/null +++ b/conformance/topologies/delta-ring-added.topology.json @@ -0,0 +1,34 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 2, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000003000", + "0000000000007000" + ] + }, + { + "id": "c", + "tokens": [ + "0000000000002000" + ] + } + ] +} diff --git a/conformance/topologies/delta-ring-before.topology.json b/conformance/topologies/delta-ring-before.topology.json new file mode 100644 index 0000000..446ae88 --- /dev/null +++ b/conformance/topologies/delta-ring-before.topology.json @@ -0,0 +1,28 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 1, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000003000", + "0000000000007000" + ] + } + ] +} diff --git a/conformance/topologies/delta-ring-removed.topology.json b/conformance/topologies/delta-ring-removed.topology.json new file mode 100644 index 0000000..0a403ff --- /dev/null +++ b/conformance/topologies/delta-ring-removed.topology.json @@ -0,0 +1,28 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 3, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000003000", + "0000000000007000" + ] + } + ] +} diff --git a/conformance/topologies/delta-ring-unaligned.topology.json b/conformance/topologies/delta-ring-unaligned.topology.json new file mode 100644 index 0000000..2b87961 --- /dev/null +++ b/conformance/topologies/delta-ring-unaligned.topology.json @@ -0,0 +1,33 @@ +{ + "formatVersion": "1.0", + "topologyId": "delta-ring", + "epoch": 4, + "replication": { + "factor": 2 + }, + "strategy": { + "kind": "ring", + "tokenAssignment": "explicit" + }, + "nodes": [ + { + "id": "a", + "tokens": [ + "0000000000001000", + "0000000000005000" + ] + }, + { + "id": "b", + "tokens": [ + "0000000000007000" + ] + }, + { + "id": "c", + "tokens": [ + "0000000000002000" + ] + } + ] +} diff --git a/conformance/topologies/lineage-directory-before.topology.json b/conformance/topologies/lineage-directory-before.topology.json new file mode 100644 index 0000000..9bfeb76 --- /dev/null +++ b/conformance/topologies/lineage-directory-before.topology.json @@ -0,0 +1,39 @@ +{ + "formatVersion": "1.0", + "topologyId": "lineage-directory", + "epoch": 1, + "replication": { + "factor": 1 + }, + "strategy": { + "kind": "directory", + "entries": [ + { + "match": { + "kind": "prefix", + "value": "ab" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "cd" + }, + "nodes": [ + "d2" + ] + } + ] + }, + "nodes": [ + { + "id": "d1" + }, + { + "id": "d2" + } + ] +} diff --git a/conformance/topologies/lineage-directory-fresh.topology.json b/conformance/topologies/lineage-directory-fresh.topology.json new file mode 100644 index 0000000..b60b25d --- /dev/null +++ b/conformance/topologies/lineage-directory-fresh.topology.json @@ -0,0 +1,48 @@ +{ + "formatVersion": "1.0", + "topologyId": "lineage-directory", + "epoch": 3, + "replication": { + "factor": 1 + }, + "strategy": { + "kind": "directory", + "entries": [ + { + "match": { + "kind": "prefix", + "value": "ab" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "cd" + }, + "nodes": [ + "d2" + ] + }, + { + "match": { + "kind": "prefix", + "value": "ef" + }, + "nodes": [ + "d1" + ] + } + ] + }, + "nodes": [ + { + "id": "d1" + }, + { + "id": "d2" + } + ] +} diff --git a/conformance/topologies/lineage-directory-refined.topology.json b/conformance/topologies/lineage-directory-refined.topology.json new file mode 100644 index 0000000..39f8c44 --- /dev/null +++ b/conformance/topologies/lineage-directory-refined.topology.json @@ -0,0 +1,57 @@ +{ + "formatVersion": "1.0", + "topologyId": "lineage-directory", + "epoch": 2, + "replication": { + "factor": 1 + }, + "strategy": { + "kind": "directory", + "entries": [ + { + "match": { + "kind": "prefix", + "value": "ab0" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "ab1" + }, + "nodes": [ + "d2" + ] + }, + { + "match": { + "kind": "prefix", + "value": "ab" + }, + "nodes": [ + "d1" + ] + }, + { + "match": { + "kind": "prefix", + "value": "cd" + }, + "nodes": [ + "d2" + ] + } + ] + }, + "nodes": [ + { + "id": "d1" + }, + { + "id": "d2" + } + ] +} diff --git a/conformance/vectors/errors/taxonomy.json b/conformance/vectors/errors/taxonomy.json index ca842e0..a65fd4a 100644 --- a/conformance/vectors/errors/taxonomy.json +++ b/conformance/vectors/errors/taxonomy.json @@ -169,7 +169,9 @@ "strategyUnsupported", "destinationOutsidePlacementSet", "policyInvalid", - "topologyMismatch" + "topologyMismatch", + "unalignedLineage", + "lineageUnsupported" ] }, { @@ -190,7 +192,8 @@ "unverified", "residue", "undetermined", - "rollbackFailed" + "rollbackFailed", + "undivided" ] } ], diff --git a/conformance/vectors/migration/lineage.json b/conformance/vectors/migration/lineage.json new file mode 100644 index 0000000..4edcebc --- /dev/null +++ b/conformance/vectors/migration/lineage.json @@ -0,0 +1,326 @@ +{ + "vectorSet": "migration-lineage", + "kind": "lineage", + "level": "migration", + "description": "The lineage classification over pairs of documents: a ring extent divided, a ring extent folded, the identity lineage under `slot`, the unaligned boundary that is refused, the `directory` prefix refined and the fresh extent beside it, the incomparable pair, and the kind that enumerates no shard.", + "requirements": [ + "DIR-002", + "DIR-010", + "ERR-050", + "LIN-004", + "LIN-006", + "LIN-007", + "LIN-011", + "LIN-012", + "LIN-013", + "LIN-014", + "LIN-016", + "LIN-021", + "LIN-022", + "LIN-031", + "LIN-033", + "MOVE-241", + "PLACE-065", + "TOPO-231" + ], + "cases": [ + { + "name": "ring-extent-divided", + "requirements": [ + "LIN-004", + "LIN-011", + "LIN-021", + "LIN-031", + "LIN-033" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-added.topology.json", + "note": "`LIN-011`: the added token divides `(1000, 3000]` into `(1000, 2000]` and `(2000, 3000]`, so both shards of the later snapshot are `divided` from one parent and the shard whose extent did not move is `moved` because its replica set did.", + "expect": { + "lineage": [ + { + "shard": "0000000000001000", + "class": "moved", + "parents": [ + "0000000000001000" + ] + }, + { + "shard": "0000000000002000", + "class": "divided", + "parents": [ + "0000000000003000" + ] + }, + { + "shard": "0000000000003000", + "class": "divided", + "parents": [ + "0000000000003000" + ] + }, + { + "shard": "0000000000005000", + "class": "unchanged", + "parents": [ + "0000000000005000" + ] + }, + { + "shard": "0000000000007000", + "class": "unchanged", + "parents": [ + "0000000000007000" + ] + } + ] + } + }, + { + "name": "ring-extent-folded", + "requirements": [ + "LIN-004", + "LIN-011", + "LIN-021", + "LIN-033" + ], + "before": "topologies/delta-ring-added.topology.json", + "after": "topologies/delta-ring-removed.topology.json", + "note": "`LIN-021`: removing the token folds `(1000, 2000]` into the extent that follows it, so the later shard is `merged` from two parents and the shard only the earlier snapshot enumerates is `folded` and follows every later entry under `LIN-033`.", + "expect": { + "lineage": [ + { + "shard": "0000000000001000", + "class": "moved", + "parents": [ + "0000000000001000" + ] + }, + { + "shard": "0000000000003000", + "class": "merged", + "parents": [ + "0000000000002000", + "0000000000003000" + ] + }, + { + "shard": "0000000000005000", + "class": "unchanged", + "parents": [ + "0000000000005000" + ] + }, + { + "shard": "0000000000007000", + "class": "unchanged", + "parents": [ + "0000000000007000" + ] + }, + { + "shard": "0000000000002000", + "class": "folded", + "parents": [ + "0000000000003000" + ] + } + ] + } + }, + { + "name": "directory-prefix-refined", + "requirements": [ + "LIN-013", + "LIN-016", + "LIN-021", + "DIR-002", + "PLACE-065" + ], + "before": "topologies/lineage-directory-before.topology.json", + "after": "topologies/lineage-directory-refined.topology.json", + "note": "`LIN-016`: refining `prefix:ab` into `ab0` and `ab1` leaves `ab` winning the keys neither longer prefix claims, so all three shards of the later table are `divided` from the one entry and the untouched `cd` is `unchanged`.", + "expect": { + "lineage": [ + { + "shard": "prefix:616230", + "class": "divided", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:616231", + "class": "divided", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:6162", + "class": "divided", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:6364", + "class": "unchanged", + "parents": [ + "prefix:6364" + ] + } + ] + } + }, + { + "name": "directory-fresh-extent", + "requirements": [ + "LIN-013", + "LIN-016", + "LIN-021", + "DIR-010" + ], + "before": "topologies/lineage-directory-before.topology.json", + "after": "topologies/lineage-directory-fresh.topology.json", + "note": "`LIN-021`: the added entry wins keys the earlier table matched to no shard under `DIR-010`, so the shard it names is `fresh` and has no parent to draw from.", + "expect": { + "lineage": [ + { + "shard": "prefix:6162", + "class": "unchanged", + "parents": [ + "prefix:6162" + ] + }, + { + "shard": "prefix:6364", + "class": "unchanged", + "parents": [ + "prefix:6364" + ] + }, + { + "shard": "prefix:6566", + "class": "fresh", + "parents": [] + } + ] + } + }, + { + "name": "slot-identity", + "requirements": [ + "LIN-007", + "LIN-012", + "LIN-021" + ], + "before": "topologies/delta-before.topology.json", + "after": "topologies/delta-moved.topology.json", + "note": "`LIN-012`: `TOPO-231` holds `slotCount` equal, so the two snapshots enumerate the same shards, the lineage is the identity of `LIN-007`, and a shard is `moved` or `unchanged` and never divided, merged, fresh, or vacated.", + "expect": { + "lineage": [ + { + "shard": "0", + "class": "unchanged", + "parents": [ + "0" + ] + }, + { + "shard": "1", + "class": "unchanged", + "parents": [ + "1" + ] + }, + { + "shard": "2", + "class": "unchanged", + "parents": [ + "2" + ] + }, + { + "shard": "3", + "class": "moved", + "parents": [ + "3" + ] + }, + { + "shard": "4", + "class": "moved", + "parents": [ + "4" + ] + }, + { + "shard": "5", + "class": "moved", + "parents": [ + "5" + ] + } + ] + } + }, + { + "name": "ring-unaligned-boundary", + "requirements": [ + "LIN-021", + "LIN-022", + "ERR-050" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-unaligned.topology.json", + "note": "`LIN-022`: a boundary that moves without dividing or folding an extent whole has no correspondence to name, so the plan is refused and the authority publishes the change as a division epoch followed by a fold epoch.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "unalignedLineage" + } + } + }, + { + "name": "incomparable-shard-identity", + "requirements": [ + "LIN-006", + "TOPO-231", + "ERR-050" + ], + "before": "topologies/delta-before.topology.json", + "after": "topologies/delta-other-seed.topology.json", + "note": "`LIN-006`: a lineage joins two snapshots on their extents and an ownership delta joins them on their identifiers, and a change that renames every shard leaves neither one an answer, so both refuse on the condition `TOPO-231` states.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "incomparableShards" + } + } + }, + { + "name": "rendezvous-has-no-extent", + "requirements": [ + "LIN-014", + "MOVE-241", + "ERR-050" + ], + "before": "topologies/rendezvous-plain.topology.json", + "after": "topologies/rendezvous-plain.topology.json", + "note": "`LIN-014`: `PLACE-032` makes `shards` empty under `rendezvous`, so the kind has no extent and no lineage, and `MOVE-251` refuses the plan before one is reached.", + "expect": { + "lineageComputed": false, + "condition": { + "code": 401, + "name": "planRefused", + "cause": "strategyUnsupported" + } + } + } + ] +} diff --git a/conformance/vectors/migration/plan-construction.json b/conformance/vectors/migration/plan-construction.json new file mode 100644 index 0000000..1a56872 --- /dev/null +++ b/conformance/vectors/migration/plan-construction.json @@ -0,0 +1,163 @@ +{ + "vectorSet": "migration-plan-construction", + "kind": "planConstruction", + "level": "migration", + "description": "How a plan derives a handoff from a lineage: the source of a divided shard, the destination of a fold that the ownership delta does not report, the local steps beside them and their order, and the plan under the identity lineage.", + "requirements": [ + "LIN-041", + "LIN-042", + "LIN-044", + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" + ], + "cases": [ + { + "name": "divided-source-from-the-parent", + "requirements": [ + "LIN-041", + "LIN-042", + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-added.topology.json", + "note": "`LIN-041`: the contents of the divided shard `0000000000002000` are held by the replicas of its parent `0000000000003000`, so the handoff names one of them as its source. A plan built over the ownership delta alone has no entry for the parent, whose replica set did not change, and names a source equal to the destination.", + "expect": { + "handoffs": [ + { + "shard": "0000000000001000", + "sourceShard": "0000000000001000", + "kind": "handoff", + "source": "b", + "destination": "c", + "id": "h-0000000000001000" + }, + { + "shard": "0000000000002000", + "sourceShard": "0000000000003000", + "kind": "divide", + "source": "b", + "destination": "b", + "id": "l-0000000000002000" + }, + { + "shard": "0000000000002000", + "sourceShard": "0000000000003000", + "kind": "handoff", + "source": "a", + "destination": "c", + "id": "h-0000000000002000-1" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "divide", + "source": "b", + "destination": "b", + "id": "l-0000000000003000" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "divide", + "source": "a", + "destination": "a", + "id": "l-0000000000003000-1" + } + ] + } + }, + { + "name": "folded-destination-outside-the-delta", + "requirements": [ + "LIN-041", + "LIN-044", + "LIN-045", + "LIN-051", + "LIN-052", + "LIN-057" + ], + "before": "topologies/delta-ring-added.topology.json", + "after": "topologies/delta-ring-removed.topology.json", + "note": "`LIN-044`: the shard that absorbs the folded extent keeps its replica set, so the ownership delta reports no entry for it, and a plan built over the delta alone moves nothing to the replica that does not hold the folded parent.", + "expect": { + "handoffs": [ + { + "shard": "0000000000001000", + "sourceShard": "0000000000001000", + "kind": "handoff", + "source": "c", + "destination": "b", + "id": "h-0000000000001000" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000002000", + "kind": "handoff", + "source": "c", + "destination": "a", + "id": "h-0000000000003000" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "combine", + "source": "b", + "destination": "b", + "id": "l-0000000000003000-1" + }, + { + "shard": "0000000000003000", + "sourceShard": "0000000000003000", + "kind": "combine", + "source": "a", + "destination": "a", + "id": "l-0000000000003000-2" + } + ] + } + }, + { + "name": "slot-plan-over-the-identity-lineage", + "requirements": [ + "LIN-041", + "LIN-045" + ], + "before": "topologies/delta-before.topology.json", + "after": "topologies/delta-moved.topology.json", + "note": "`LIN-007`: under the identity lineage every shard is its own parent, so the plan is the one the ownership delta already produced and this case is the regression guard for it.", + "expect": { + "handoffs": [ + { + "shard": "3", + "sourceShard": "3", + "kind": "handoff", + "source": "b", + "destination": "d", + "id": "h-3" + }, + { + "shard": "4", + "sourceShard": "4", + "kind": "handoff", + "source": "b", + "destination": "d", + "id": "h-4" + }, + { + "shard": "5", + "sourceShard": "5", + "kind": "handoff", + "source": "b", + "destination": "d", + "id": "h-5" + } + ] + } + } + ] +} diff --git a/conformance/vectors/observability/inventory-migration.json b/conformance/vectors/observability/inventory-migration.json index fe4efcc..28145cc 100644 --- a/conformance/vectors/observability/inventory-migration.json +++ b/conformance/vectors/observability/inventory-migration.json @@ -81,10 +81,11 @@ "note": "`OBS-020` names each event, the severity it carries, and the payload members it carries beyond the common members of `OBS-021`. A port may carry further members and may not rename one of these. A `deduplication` member appears on the four events `OBS-024` names and on no other.", "expect": { "surface": "migration", - "eventCount": 9, + "eventCount": 10, "names": [ "sharder.migration.cutover_committed", "sharder.migration.failed", + "sharder.migration.lineage", "sharder.migration.planned", "sharder.migration.quiesce_expired", "sharder.migration.rebase_pending", @@ -94,6 +95,20 @@ "sharder.migration.superseded" ], "events": [ + { + "name": "sharder.migration.lineage", + "severity": "info", + "payload": [ + "unchanged", + "moved", + "divided", + "merged", + "fresh", + "split", + "folded", + "vacated" + ] + }, { "name": "sharder.migration.planned", "severity": "info", diff --git a/conformance/vectors/topology/ownership-delta.json b/conformance/vectors/topology/ownership-delta.json index 8c7b3ad..163f4ea 100644 --- a/conformance/vectors/topology/ownership-delta.json +++ b/conformance/vectors/topology/ownership-delta.json @@ -2,12 +2,13 @@ "vectorSet": "topology-ownership-delta", "kind": "ownershipDelta", "level": "core", - "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, and the changes that make shard identity incomparable.", + "description": "The ownership delta between two snapshots, the reorder-only case that gains and loses nothing, the entry order over sixteen slots, the replica set under a shortfall, the `ring` pair whose two snapshots enumerate different shard sets, and the changes that make shard identity incomparable.", "requirements": [ "ERR-010", "PLACE-031", "REPL-017", "REPL-020", + "RING-031", "SEC-013", "SLOT-031", "SPREAD-014", @@ -333,6 +334,100 @@ ] } }, + { + "name": "ring-token-added", + "requirements": [ + "TOPO-211", + "TOPO-213", + "PLACE-031", + "RING-031" + ], + "before": "topologies/delta-ring-before.topology.json", + "after": "topologies/delta-ring-added.topology.json", + "note": "`TOPO-213`: the added token divides the extent `0000000000003000` bounded, so the second snapshot enumerates `0000000000002000` and the first does not. That shard's entry carries an empty before set and reports every node it gained.", + "expect": { + "shardsChanged": 2, + "delta": [ + { + "shard": "0000000000001000", + "before": [ + "a", + "b" + ], + "after": [ + "a", + "c" + ], + "gained": [ + "c" + ], + "lost": [ + "b" + ] + }, + { + "shard": "0000000000002000", + "before": [], + "after": [ + "c", + "b" + ], + "gained": [ + "b", + "c" + ], + "lost": [] + } + ] + } + }, + { + "name": "ring-token-removed", + "requirements": [ + "TOPO-211", + "TOPO-213", + "PLACE-031", + "RING-031" + ], + "before": "topologies/delta-ring-added.topology.json", + "after": "topologies/delta-ring-removed.topology.json", + "note": "`TOPO-213`: removing the token folds `0000000000002000` into the extent that follows it, so only the first snapshot enumerates that shard. Its entry carries an empty after set and follows every entry for a shard the second snapshot enumerates.", + "expect": { + "shardsChanged": 2, + "delta": [ + { + "shard": "0000000000001000", + "before": [ + "a", + "c" + ], + "after": [ + "a", + "b" + ], + "gained": [ + "b" + ], + "lost": [ + "c" + ] + }, + { + "shard": "0000000000002000", + "before": [ + "c", + "b" + ], + "after": [], + "gained": [], + "lost": [ + "b", + "c" + ] + } + ] + } + }, { "name": "replica-prefix-short-of-factor", "requirements": [ diff --git a/docs/README.md b/docs/README.md index 7b30082..e012b1d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -197,10 +197,10 @@ other port carries too. | placement and weights | [0002](design/adr/0002-placement-strategy-set.md), [0003](design/adr/0003-integer-node-weights.md), [0010](design/adr/0010-shard-identifier-naming.md), [0011](design/adr/0011-derived-assignment-virtual-nodes.md), [0013](design/adr/0013-range-bounds-over-routing-key.md), [0039](design/adr/0039-placement-cost-model-and-warning-thresholds.md), [0043](design/adr/0043-assignment-mode-defaults.md), [0054](design/adr/0054-range-strategy-withdrawal.md), [0055](design/adr/0055-slot-derived-assignment-withdrawal.md), [0071](design/adr/0071-candidate-ordering-over-a-placement-ring.md), [0072](design/adr/0072-preparation-cost-reported-before-preparation.md) | | topology model and format | [0004](design/adr/0004-topology-provider-contract.md), [0005](design/adr/0005-epoch-and-version-semantics.md), [0006](design/adr/0006-failure-domain-model.md), [0008](design/adr/0008-json-canonical-serialisation.md), [0009](design/adr/0009-override-composition.md), [0038](design/adr/0038-provider-contract-in-the-specification.md), [0048](design/adr/0048-conditional-fetch-in-the-provider-contract.md) | | replication and failover | [0007](design/adr/0007-administrative-state-and-health-state.md), [0012](design/adr/0012-balance-bound-tolerances.md), [0014](design/adr/0014-read-affinity-as-a-separate-call.md), [0015](design/adr/0015-spread-degradation-algorithm.md), [0016](design/adr/0016-node-health-state-machine.md), [0017](design/adr/0017-failover-depth-and-substitution.md), [0036](design/adr/0036-spread-relaxation-ladder-direction.md), [0042](design/adr/0042-domain-path-scope.md), [0045](design/adr/0045-attempt-limit-resolution-order.md), [0046](design/adr/0046-bounded-routing-decision-surface.md), [0063](design/adr/0063-decision-api-surface-boundaries.md), [0069](design/adr/0069-per-level-occupancy-cap.md), [0070](design/adr/0070-spread-stage-feasibility-from-a-domain-count.md), [0075](design/adr/0075-affinity-path-length-refusal.md) | -| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md) | +| change, handoff, and fencing | [0018](design/adr/0018-concurrent-ownership-during-handoff.md), [0019](design/adr/0019-handoff-coordination-and-recovery.md), [0020](design/adr/0020-recipient-side-fencing-verdicts.md), [0021](design/adr/0021-migration-backpressure-control.md), [0022](design/adr/0022-range-split-lineage.md), [0023](design/adr/0023-snapshot-visibility-and-thread-ownership.md), [0044](design/adr/0044-ownership-under-an-identity-mismatch.md), [0047](design/adr/0047-redirect-walk-under-the-retry-budget.md), [0049](design/adr/0049-ownership-delta-computed-on-demand.md), [0050](design/adr/0050-plan-rebase-onto-a-newer-snapshot.md), [0051](design/adr/0051-recovery-from-an-undetermined-cutover.md), [0056](design/adr/0056-advisory-cutover-withdrawal.md), [0057](design/adr/0057-step-budget-adjustment-withdrawal.md), [0073](design/adr/0073-prepared-placement-retention-multiple.md), [0074](design/adr/0074-quiesce-lease-margin-and-clock-assumption.md), [0086](design/adr/0086-shard-lineage-derived-from-extent.md), [0087](design/adr/0087-lineage-capability-derived-rather-than-declared.md), [0089](design/adr/0089-a-division-undone-by-a-combination.md), [0090](design/adr/0090-no-rebase-across-a-lineage-boundary.md), [0091](design/adr/0091-lineage-classification-as-a-signal.md), [0092](design/adr/0092-an-observability-sink-for-the-coordinator.md) | | cross-cutting contracts | [0024](design/adr/0024-closed-numbered-error-taxonomy.md), [0025](design/adr/0025-observability-contract-and-explain-record.md), [0026](design/adr/0026-configuration-defaults-and-locality.md), [0027](design/adr/0027-hash-seed-exposure-and-tenancy.md), [0060](design/adr/0060-node-label-cardinality-ceiling.md), [0076](design/adr/0076-ordered-rows-in-a-precedence-table.md) | | Java binding | [0028](design/adr/0028-java-module-and-artifact-layout.md), [0029](design/adr/0029-exception-idiom-for-the-taxonomy.md), [0031](design/adr/0031-jdk-baseline.md), [0032](design/adr/0032-dependency-free-json-and-canonicalisation.md), [0033](design/adr/0033-opaque-identifier-value-types.md), [0034](design/adr/0034-lazy-candidate-traversal-surface.md), [0041](design/adr/0041-exact-product-comparison-surface.md), [0081](design/adr/0081-single-java-module.md), [0085](design/adr/0085-hook-declarations-and-refused-aborts.md) | -| conformance | [0035](design/adr/0035-manifest-driven-conformance-harness.md), [0037](design/adr/0037-specification-defect-repairs.md), [0052](design/adr/0052-conformance-level-partition.md), [0058](design/adr/0058-conformance-surfaces.md), [0059](design/adr/0059-place-conformance-level.md), [0061](design/adr/0061-suite-revision-identifier.md), [0064](design/adr/0064-hash-verification-before-generation.md), [0065](design/adr/0065-level-coverage-inside-surface-boundaries.md), [0077](design/adr/0077-scale-conformance-level.md), [0078](design/adr/0078-observability-contract-as-data.md), [0084](design/adr/0084-missing-member-as-a-validation-error.md) | +| conformance | [0035](design/adr/0035-manifest-driven-conformance-harness.md), [0037](design/adr/0037-specification-defect-repairs.md), [0052](design/adr/0052-conformance-level-partition.md), [0058](design/adr/0058-conformance-surfaces.md), [0059](design/adr/0059-place-conformance-level.md), [0061](design/adr/0061-suite-revision-identifier.md), [0064](design/adr/0064-hash-verification-before-generation.md), [0065](design/adr/0065-level-coverage-inside-surface-boundaries.md), [0077](design/adr/0077-scale-conformance-level.md), [0078](design/adr/0078-observability-contract-as-data.md), [0084](design/adr/0084-missing-member-as-a-validation-error.md), [0088](design/adr/0088-lineage-inside-the-migration-surface.md) | | requirement identifiers | [0053](design/adr/0053-requirement-withdrawal-convention.md) | | ports and declarations | [0079](design/adr/0079-repository-layout-for-multiple-ports.md), [0080](design/adr/0080-conformance-declaration-format.md), [0082](design/adr/0082-continuous-integration-and-dependency-updates.md) | | release staging | [0083](design/adr/0083-publication-as-the-last-stage.md) | diff --git a/docs/design/05-glossary.md b/docs/design/05-glossary.md index 27b4cea..d169226 100644 --- a/docs/design/05-glossary.md +++ b/docs/design/05-glossary.md @@ -25,6 +25,22 @@ the design states, and the Java port renders each one. lowercase hexadecimal, so the strategy enumerates no shard extents. - **shard identifier**. The stable name of a shard within a topology, unique across the topology document. +- **extent**. The set of routing keys a snapshot maps to one shard. An extent is determined by the + topology document alone, never by sampling keys, and the extents of one snapshot are pairwise + disjoint. Under `ring` an extent is a token range, under `slot` it is the keys whose remainder is + one slot index, and under `directory` it is a matcher narrowed by the precedence of the entries + that outrank it. +- **lineage**. The correspondence between the extents of two snapshots of one topology, which + answers where a shard's contents come from when an epoch changes which shards exist. A lineage is + derived from the two documents rather than recorded in either, and it is the identity wherever + the two snapshots enumerate the same shards. +- **split**. A change in which one shard's extent becomes the extents of two or more shards of the + next epoch. The authority publishes the change; the library classifies it and plans it. +- **merge**. A change in which the extents of two or more shards become the extent of one shard of + the next epoch. A merge is the reverse of a split. +- **local step**. The part of a split or a merge that happens on one node, where that node holds + the parent under the earlier snapshot and the child under the later one. Nothing moves between + nodes; the node divides or folds its own copy so that what it holds matches the extent it owns. - **partition**. A synonym for shard in external literature. The corpus uses shard. - **slot**. A shard produced by dividing the keyspace into a fixed count of numbered parts by modular arithmetic over the key hash. diff --git a/docs/design/10-specification.md b/docs/design/10-specification.md index 55f9dce..d5bb067 100644 --- a/docs/design/10-specification.md +++ b/docs/design/10-specification.md @@ -218,6 +218,7 @@ the rule the Withdrawn identifiers section gives. | `READ` | read routing and read affinity | Replication and failover | `readAffinity` | | `TOPO` | load pipeline, snapshots, ownership delta | Topology change and rebalancing | `routing` | | `FENCE` | fencing tokens and recipient verdicts | Topology change and rebalancing | `fencing` | +| `LIN` | shard extents, the lineage relation, and its classification | Topology change and rebalancing | `migration` | | `MOVE` | handoff states, hooks, coordination | Topology change and rebalancing | `migration` | | `RATE` | migration concurrency and backpressure | Topology change and rebalancing | `migration` | | `ERR` | the closed set of failure conditions | Error taxonomy | stated above | @@ -2912,6 +2913,230 @@ redirect is never a first attempt, so `FAIL-032` does not exempt one. Where the redirect, an implementation MUST count the refusal in `sharder.attempts.retries_refused` and MUST NOT count the redirect in `sharder.fencing.redirects`. +### Shard lineage + +An ownership delta joins two snapshots on the shard identifier. Where a shard carries the same +identifier under both, that join answers which nodes gained and lost it. Where an epoch changes +which shards exist, the join has no answer for the shards whose identifiers appeared or vanished, +and the contents of those shards still have to come from somewhere. + +A lineage is the second join, over the keys a shard holds rather than over its name. It is derived +from the topology document, it is consulted only where the two snapshots enumerate different shard +sets, and it belongs to the `migration` surface: an implementation that exposes no handoff +coordinator computes none. + +#### Shard extent + +`LIN-001`. The **extent** of a shard under a snapshot MUST be the set of routing keys that +`shardOf` maps to that shard under that snapshot. An extent MUST be determined from the topology +document alone. An implementation MUST NOT compute an extent by sampling routing keys, evaluating +`shardOf` over a generated set, or reading any value outside the document. + +`LIN-002`. Two extents are equal when they admit the same routing keys, are disjoint when they +admit no routing key in common, and the first contains the second when every routing key the second +admits is one the first admits. An implementation MUST decide each of the three from the +document rather than by enumerating keys, and `LIN-011` through `LIN-014` state how under each +strategy kind. + +`LIN-003`. The extents of the shards one snapshot enumerates MUST be pairwise disjoint. A routing +key that `shardOf` maps to no shard, which under `directory` is the no-match of `DIR-010`, belongs +to no extent, and an implementation MUST NOT treat it as belonging to one. + +#### Lineage of two snapshots + +`LIN-004`. The **lineage** of an earlier snapshot and a later snapshot MUST be the correspondence +their extents induce: for each shard of the later snapshot, the shards of the earlier one whose +extents meet its extent, and for each shard of the earlier snapshot, the shards of the later one +whose extents meet its extent. A lineage relates extents and MUST NOT be read from, or recorded in, +any member of a topology document. + +`LIN-005`. Computing a lineage MUST be a pure function of the two snapshots. It MUST NOT read +health, a clock, a handoff state, or any ownership delta. + +`LIN-006`. An implementation MUST refuse to compute a lineage between two snapshots whose shard +identity is not comparable under `TOPO-231`, and MUST report the refusal as `TOPO-231` states. +Comparability is the same condition for both joins, because a change that renames every shard +leaves neither one an answer. + +`LIN-007`. Where the two snapshots enumerate the same set of shard identifiers, the lineage MUST be +the identity: each shard's parent and child are that shard alone, and no shard is classified +`divided`, `merged`, `fresh`, `split`, `folded`, or `vacated` under `LIN-021`. Under every strategy kind the shard +identifier of `PLACE-031` renders the geometry that fixes the extent, so an equal shard set is an +equal extent set: under `ring` the identifier is the owning token, under `slot` it is the slot index +at a `slotCount` that `TOPO-231` holds equal, and under `directory` it is the matcher itself, whose +effective extent is fixed by the entry set that the shard set enumerates. + +#### Extents under each strategy + +`LIN-011`. Under `ring`, the extent of the shard whose identifier decodes to the token `t` MUST be +the half-open interval of routing key hash values from the greatest token below `t` to `t`, +ascending as unsigned 64-bit values and wrapping at zero, which is the interval `RING-020` gives the +owning ring entry. Containment and equality MUST be decided by comparing interval bounds as unsigned +64-bit values. + +`LIN-012`. Under `slot`, `TOPO-231` refuses a pair whose `slotCount` differs, so the two snapshots +enumerate the same slots and every extent is equal to its counterpart. The lineage under `slot` MUST +therefore be the identity of `LIN-007`. + +`LIN-013`. Under `directory`, the extent of a shard MUST be the keys its entry wins under the +precedence of `PLACE-065`, which is its matcher narrowed by every entry of the same table that +outranks it. An implementation MUST decide equality, containment, and disjointness of two directory +extents over the prefix trie of the two tables' decoded matcher values, and MUST NOT decide them by +evaluating the tables over generated keys. + +`LIN-016`. The trie of `LIN-013` MUST be read as follows. Every decoded matcher value of either +table is a node, and the empty value is a node. Each node contributes two regions of the keyspace: +the key equal to that node, and the keys strictly extending it that no deeper node is a prefix of. +Two keys of one region match the same entries of either table, so a region is the finest distinction +either table draws, and the regions together admit every key. The extent of a shard is the set of +regions its entry wins, and an implementation MUST compare two extents as those sets. An `exact` +matcher wins only the region that is its own key, because a region of keys strictly extending a node +contains no node and every matcher value is one. + +`LIN-014`. Under `rendezvous`, `PLACE-032` makes `shards` empty and `MOVE-241` excludes the kind +from orchestrated migration, so there is no extent and no lineage. An implementation MUST refuse a +plan under `MOVE-251` before it reaches a lineage. + +`LIN-015`. A registered strategy outside the core set MUST supply its own lineage. An implementation +MUST NOT derive a registered strategy's extents by sampling routing keys and MUST NOT infer a +lineage from the strategy's name. Where the two snapshots enumerate different shard sets and the +strategy supplies no lineage, an implementation MUST refuse the plan under `ERR-050` with the cause +`lineageUnsupported`. + +#### Lineage classification + +`LIN-021`. An implementation MUST classify each shard of each snapshot against the other as exactly +one of these. + +A shard the later snapshot enumerates is classified against the earlier one, and a shard only the +earlier snapshot enumerates is classified against the later one, so every shard of the lineage +carries exactly one class. + +| Class | Snapshot | Condition | +|---|---|---| +| `unchanged` | later | one parent, whose extent is equal, and the replica set is equal | +| `moved` | later | one parent, whose extent is equal, and the replica set differs | +| `divided` | later | one parent, whose extent strictly contains this one | +| `merged` | later | two or more parents, whose extents this one contains | +| `fresh` | later | no parent, this extent meeting no extent of the earlier snapshot | +| `split` | earlier only | two or more children, whose extents this one contains | +| `folded` | earlier only | one child, whose extent strictly contains this one | +| `vacated` | earlier only | no child, this extent meeting no extent of the later snapshot | +| `unaligned` | either | an extent of the other snapshot that this one neither contains nor is contained by | + +`LIN-022`. An implementation MUST refuse a plan over a lineage that classifies any shard +`unaligned`, and MUST report the refusal under `ERR-050` with the cause `unalignedLineage` naming +the shard. A boundary that moves without either dividing or folding an extent whole has no +correspondence an implementation can name, and an authority publishes such a change as two epochs +instead, the first dividing every extent the change crosses and the second folding the pieces into +their destinations. Each of the two is plannable on its own. + +`LIN-023`. A classification MUST be a pure function of the two snapshots under `LIN-005`, and an +implementation MUST NOT let the order in which it enumerates shards change any shard's class. + +#### The lineage operation + +`LIN-031`. An implementation that exposes the `migration` surface MUST expose the lineage between +two snapshots of the same `topologyId` as an operation the integrator calls, answering each shard's +class under `LIN-021` and, for a shard the later snapshot enumerates, the shards of the earlier one +its extent draws from. An integrator sizes a migration before deciding to run it, which is the +reason `TOPO-212` makes the ownership delta a call rather than a product of installation. + +`LIN-032`. An implementation MUST NOT compute a lineage as a step of `TOPO-001`, as part of +installing a snapshot, or as a precondition of a routing call reading an installed snapshot. + +`LIN-033`. The entries of a lineage MUST be ordered as `TOPO-213` orders an ownership delta: first +the entries for shards the later snapshot enumerates, in the order `shards` gives for that snapshot +under `PLACE-031`, then the entries for shards only the earlier snapshot enumerates, in the order +`shards` gives for it. + +`LIN-034`. Computing a lineage MUST cost no more than one walk of each snapshot's shard enumeration +and MUST NOT evaluate a candidate ordering. Under `ring` the two enumerations are ascending token +orders under `RING-031`, so the correspondence is a merge of two ordered sequences, and an +implementation MUST NOT materialise either enumeration where its shard count is the figure +`PLACE-075` bounds a delta over. + +#### Plan construction over a lineage + +`LIN-041`. `plan` MUST derive its handoffs from the lineage rather than from the ownership delta +alone, one parent at a time. For each shard `s` the later snapshot enumerates, for each shard `p` +whose extent `s` draws from under `LIN-004`, and for each node `d` of the replica set of `s` under +the later snapshot that is not an entry of the replica set of `p` under the earlier snapshot, a plan +MUST carry one handoff moving `p`'s contents for `s` to `d`. A destination already holding a +parent's contents needs no handoff for that parent, and a destination holding one parent's contents +still needs a handoff for every other parent it does not hold. + +`LIN-042`. An implementation MUST NOT emit a handoff whose source and destination are the same node. +A handoff names the node that holds the contents and the node that is to hold them, and a node +cannot be both. + +`LIN-043`. A shard classified `fresh` has no parent and no contents to move, so a plan MUST NOT emit +a handoff for it. Ownership of a fresh shard is established by the epoch change alone. + +`LIN-044`. A plan MUST include a handoff for a shard whose extent grew even where its replica set is +unchanged and the ownership delta therefore reports no entry for it. `TOPO-211` answers which shards +changed owner, which is a different question from which shards must receive contents, and a plan +built over the delta alone omits the destination of every fold whose replica set happens not to +change. + +`LIN-045`. The source of the handoff `LIN-041` admits MUST be chosen as follows. Let the departing +replicas of `p` be the entries of `p`'s replica set under the earlier snapshot that are not entries +of `s`'s replica set under the later snapshot, in the order the earlier snapshot gives, and let `i` +be the position of `d` among the destinations that parent admits, in the order the later snapshot +gives. The source MUST be the departing replica at position `i` where there is one, and otherwise +the first entry of `p`'s replica set under the earlier snapshot. A replica that is giving the shard +up is drained in preference to one that is keeping it, and a plan is a pure function of the two +snapshots and the policy under `MOVE-221`, so the choice MUST be stated rather than left to an +implementation. + +#### The local division step + +`LIN-051`. Where a shard `s` of the later snapshot has a parent `p` whose extent differs from +`s`'s, and a node holds `p` under the earlier snapshot and `s` under the later one, a plan MUST +carry one local step for that node. The node already holds the contents, so nothing moves +between nodes, and what it has to do is divide or fold its own copy so that what it holds matches +the extent of `s`. A local step names the same node as its source and its destination, which is the one +case `LIN-042` does not govern, because no handoff of contents is described. + +`LIN-052`. A local step MUST call `divide` where the extent of `s` is strictly contained in the +extent of `p`, and `combine` where the extent of `s` contains the extents of two or more parents. +An implementation MUST call one local hook per local step and MUST NOT call both. + +`LIN-053`. An implementation MUST refuse a plan that requires a local step where `declare` answers +`supportsLineage` of false, reporting `planRefused` under `ERR-050` with the cause +`lineageUnsupported`. Whether an integrator's storage can divide a copy in place is a property of +that storage, which the library cannot observe, so `MOVE-111` declares it beside `supportsRollback` +and `supportsVerify` and the plan is refused rather than calling a hook that was never implemented. +A refusal at planning is the only useful moment: a plan that reached `dividing` and found no hook +would have no way forward and no way back. + +`LIN-054`. A local step MUST be retried under the same attempt rules as any other hook, and an +implementation MUST pass the attempt number in the context so a hook that already divided its copy +can answer success rather than dividing twice. An implementation MUST NOT require a local hook to +be idempotent beyond answering success for work it has already done. + +`LIN-055`. An implementation MUST NOT sequence a local step onto a node that does not hold every +shard the step names under the snapshot the step names it. A local step for `s` drawn from `p` is +sequenced only where the node's replica set membership holds under both snapshots, which is the +condition `LIN-051` states, and a plan MUST NOT admit one otherwise. + +`LIN-056`. A local step in `dividing` that is aborted MUST be undone by the inverse hook: a +division is undone by `combine` and a fold by `divide`, called through `rollback` under +`MOVE-421`. A local step is therefore reversible like every other state that reaches `aborting`, +and `MOVE-233` keeps the unconditional form it has. Where the inverse fails and its attempts are +exhausted, the handoff reaches `failed` with the kind `undivided` under `MOVE-011`. + +`LIN-057`. An implementation MUST order a local step against the handoffs of the same shard: a +division MUST be sequenced before any handoff that draws from the divided parent, and a fold MUST be +sequenced after every handoff that draws into the folded shard. A destination copying from a parent +that is being divided underneath it would copy an extent that is changing, and a node folding a copy +before the other parents have arrived would fold an incomplete one. + +`LIN-058`. A local step MUST count against the concurrency the migration policy of `RATE-011` +admits, in the same way a handoff does. A local step performs storage work on a node, which is the +thing the policy exists to bound, and an implementation MUST NOT admit one outside +`maxConcurrentHandoffs`. + ### Shard ownership handoff A handoff moves ownership of one shard from a source node to a destination node. The library @@ -2930,6 +3155,7 @@ storage protocol. | `catchingUp` | the residue accumulated during the copy is being closed | | `cutover` | the source is quiescing and the cutover record is being committed | | `verifying` | the destination copy is being checked against the source | +| `dividing` | a node that holds both the parent and the child is dividing or folding its own copy | | `cleanup` | the source copy is being released | | `complete` | terminal, ownership moved and the source released | | `aborting` | compensation is running after an abort | @@ -2937,7 +3163,10 @@ storage protocol. | `failed` | terminal, operator action is required | `MOVE-011`. A handoff in `failed` MUST carry exactly one failure kind from the set `unverified`, -`residue`, `undetermined`, and `rollbackFailed`. +`residue`, `undetermined`, `rollbackFailed`, and `undivided`. A kind of `undivided` states that a +local step under `LIN-051` neither completed nor was undone, so the node holds a copy that matches +neither the parent's extent nor the child's. None of the other four describes that state: each names +a condition of a copy that moved between nodes, and this one never left the node it is on. #### Transitions @@ -2948,6 +3177,10 @@ storage protocol. | none | `planned` | plan construction admits the shard | | `planned` | `preparing` | the rate policy admits the handoff | | `planned` | `aborted` | abort requested, the plan is superseded, or a rebase drops the handoff | +| `planned` | `dividing` | the rate policy admits a local step under `LIN-051` | +| `dividing` | `complete` | `divide` or `combine` returns success | +| `dividing` | `aborting` | abort requested, plan superseded or rebased past it, or attempts exhausted | +| `dividing` | `failed` | `divide` or `combine` reports a permanent failure | | `preparing` | `transferring` | `prepare` returns success | | `preparing` | `aborting` | abort requested, plan superseded or rebased past it, or attempts exhausted | | `transferring` | `catchingUp` | `transfer` reports no bulk remaining | @@ -3133,9 +3366,11 @@ interface MovementHooks: cleanup(ctx) -> HookResult rollback(ctx) -> HookResult observe(ctx) -> ObserveResult + divide(ctx) -> HookResult + combine(ctx) -> HookResult HandoffContext ctx = { - shardId, topologyId, fromEpoch, toEpoch, + shardId, sourceShardId, topologyId, fromEpoch, toEpoch, source: NodeId, destination: NodeId, attempt: u32, deadlineMillis: u32 } @@ -3143,7 +3378,8 @@ HandoffContext ctx = { HookDeclaration = { budgetUnit: string, # opaque to the library supportsRollback: boolean, - supportsVerify: boolean + supportsVerify: boolean, + supportsLineage: boolean } HookResult = one of { success, deferred(retryAfterMillis), retryable(reason), @@ -3412,9 +3648,12 @@ guarantees. `MOVE-411`. An abort requested in `planned` MUST move the handoff directly to `aborted` without calling a hook. No hook has run, so nothing is to compensate. -`MOVE-421`. An abort requested in `preparing`, `transferring`, or `catchingUp` MUST move the handoff -to `aborting` and MUST call `rollback`, whose duty is to release the destination's partial copy and -any scratch state `prepare` created. The source is untouched throughout, so the abort loses nothing. +`MOVE-421`. An abort requested in `preparing`, `transferring`, `catchingUp`, or `dividing` MUST move +the handoff to `aborting` and MUST call `rollback`, whose duty is to release the destination's +partial copy and any scratch state `prepare` created. From the first three the source is untouched +throughout, so the abort loses nothing. From `dividing` there is no second copy to release, and +`rollback` MUST restore the extent the node held before the step, which `LIN-056` states is the +inverse hook. `MOVE-431`. An abort requested in `cutover` MUST be admitted only while no cutover record belonging to the handoff exists under `MOVE-102`. The coordinator MUST call `observe` before @@ -3729,10 +3968,13 @@ routing conditions and does not govern these. ### Migration conditions -`ERR-050`. `planRefused` MUST be raised by `plan` under `MOVE-081`, `MOVE-251`, and `RATE-021`, and -by `rebase` under `MOVE-095`. It MUST carry a `cause` from the closed set `incomparableShards`, -`epochNotAdvancing`, `strategyUnsupported`, `destinationOutsidePlacementSet`, `policyInvalid`, and -`topologyMismatch`. +`ERR-050`. `planRefused` MUST be raised by `plan` under `MOVE-081`, `MOVE-251`, `RATE-021`, +`LIN-013`, `LIN-015`, and `LIN-022`, and by `rebase` under `MOVE-095`. It MUST carry a `cause` from +the closed set `incomparableShards`, `epochNotAdvancing`, `strategyUnsupported`, +`destinationOutsidePlacementSet`, `policyInvalid`, `topologyMismatch`, `unalignedLineage`, and +`lineageUnsupported`. The set is closed against a binding under `ERR-062`, which forbids a leaf of +`ERR-010` that this document does not state; a cause is not a leaf, and the two this batch adds are +raised by `plan` alone. `ERR-051`. `quiesced` MUST be raised for a write to a shard between the success of `quiesce` and the return of `commitCutover`, under `MOVE-311`. It MUST be reported as retryable at both the source and @@ -3915,6 +4157,7 @@ implementation MAY carry further members beyond those named. | `health.transition` | a health state changes | `info` | `node`, `from`, `to`, `trigger` | | `health.ejection_refused` | the ceiling refuses a transition | `warning` | `node`, `ejected`, `setSize` | | `fencing.refused` | a recipient refuses a request | `warning` | `relation`, `ownership`, `currentOwner`, `token` | +| `migration.lineage` | `LIN-031` is called | `info` | `unchanged`, `moved`, `divided`, `merged`, `fresh`, `split`, `folded`, `vacated` | | `migration.planned` | `plan` returns a plan | `info` | `handoffCount`, `policy` | | `migration.state_changed` | a handoff changes state | `info` | `handoff`, `shard`, `from`, `to`, `trigger` | | `migration.cutover_committed` | a record is committed | `info` | `shard`, `source`, `destination`, `windowMillis` | diff --git a/docs/design/30-conformance.md b/docs/design/30-conformance.md index 3ecc6a4..dabaf64 100644 --- a/docs/design/30-conformance.md +++ b/docs/design/30-conformance.md @@ -493,10 +493,16 @@ assertion that a port warning about an ordinary topology fails. A port compares the inventory against its own registry and its own sink. A driver holds neither, so [`../../conformance/driver/python/run_suite.py`](../../conformance/driver/python/run_suite.py) checks the inventory's internal consistency and its surface assignment, as it does for the closed -condition set of `errorTaxonomy`. The events a running library emits are asserted where the suite -already drives one: the health scenarios carry `sharder.health.ejection_refused` in their -expectations. [`adr/0078`](adr/0078-observability-contract-as-data.md) records what is asserted and -what is not. +condition set of `errorTaxonomy`. [`adr/0078`](adr/0078-observability-contract-as-data.md) records +what is asserted and what is not. + +What the suite does not assert is emission. No driver checks that a running library emitted a named +event at the moment a requirement says it does, for any surface. A scenario carries the shape of +such an expectation in one place, the `ejectionsRefused` member the health scenarios hold, and no +driver reads that member either, so it is data rather than coverage. A port that declares an +inventory it never emits from passes every level, which is the state the Java port was in for the +whole `migration` surface. Until a driver asserts emission, a port's own tests are where emission is +checked, and the inventory is the contract the suite holds it to. ## Properties @@ -751,6 +757,39 @@ them, `MOVE-151` through `MOVE-238`, because a state sequence is an output. `MOV `MOVE-103` are the same case: a rebase is a classification of each handoff against a snapshot, and both the classification and the state it leaves are outputs. +### Shard lineage + +`LIN-021` through `LIN-045` are classifications and plans, which are outputs, so +`vectors/migration/lineage.json` and `vectors/migration/plan-construction.json` carry them. Four of +the `LIN` requirements have no executable test, for reasons this section's other entries already +give. + +`LIN-005` and `LIN-023` constrain how a lineage is produced rather than what it is: the first +forbids reading health, a clock, or a handoff state, and the second forbids the enumeration order +changing a class. An implementation that departs from either produces a different classification +wherever the departure reaches one, and fails a lineage case there. + +`LIN-032` is a rule about where an operation is not called, which is the case +`Concurrency and visibility` describes, and `LIN-034` bounds what a lineage costs, which is the case +`Placement cost` describes. `LIN-031` is not among them: it requires the lineage to be an operation +the integrator calls, so `vectors/migration/lineage.json` reaches it through the surface a port +exports rather than through the class behind it. A port that computed a lineage it could not expose +would satisfy the classification and fail the requirement that names it. + +`LIN-015` binds a registered strategy outside the core set. The suite carries core-set documents, so +no data file can present one, and the same is true of every requirement about a registered strategy. + +`LIN-053`, `LIN-054`, and `LIN-058` each read something outside the two snapshots: a hook +declaration, an attempt count, and the concurrency a policy admits. The first is the case +`Movement hooks` describes, and the other two are the case `Rate control and measurement` +describes. The state sequence a local step runs is an output, so +`scenarios/handoff-local-division.json`, `scenarios/handoff-local-fold.json`, and +`scenarios/handoff-local-step-aborted.json` carry `LIN-051`, `LIN-052`, `LIN-056`, and `LIN-057`. + +`LIN-043` states that a plan emits no handoff for a shard with no parent. A fresh extent arises +where a later snapshot admits routing keys that the earlier one matched to no shard, which is the +`directory` no-match of `DIR-010`, and `vectors/migration/lineage.json` carries that case. + ### Configuration and security `CFG-001` through `CFG-007`, `CFG-060` through `CFG-062`, and `CFG-064` govern how settings are diff --git a/docs/design/adr/0022-range-split-lineage.md b/docs/design/adr/0022-range-split-lineage.md index 08f877c..a88ce1d 100644 --- a/docs/design/adr/0022-range-split-lineage.md +++ b/docs/design/adr/0022-range-split-lineage.md @@ -7,6 +7,15 @@ The whole `SPLIT` prefix is withdrawn with the `range` strategy, which was the o the decomposition into local steps, and the `splitLocal` and `mergeLocal` hooks leave the design with it. The reasoning below is what a later minor version reads before adding the kind back. +Amended on 2026-09-19 by [`0086`](0086-shard-lineage-derived-from-extent.md). The lineage returns in +a different shape, under the `LIN` prefix and fresh identifiers, derived from each strategy's own +keyspace geometry rather than from authored range bounds. The `SPLIT` prefix stays withdrawn and +admits no further identifier. What this record got right and `0086` keeps: a lineage relates extents +rather than identifiers, a change in flight is two epochs and nothing else, an unaligned boundary +move is refused and republished as a division epoch followed by a fold epoch, and explicit lineage +members do not belong in the document. What `0086` does not take: the `range` strategy, the authored +bounds, and the rule that a division which has already succeeded cannot be undone. + ## Context The `range` strategy names shards by `shardId` and bounds them by a half-open interval. A split diff --git a/docs/design/adr/0054-range-strategy-withdrawal.md b/docs/design/adr/0054-range-strategy-withdrawal.md index d4ec059..0caa4f7 100644 --- a/docs/design/adr/0054-range-strategy-withdrawal.md +++ b/docs/design/adr/0054-range-strategy-withdrawal.md @@ -2,6 +2,14 @@ Status: accepted. Date: 2026-09-17. +Amended on 2026-09-19 by [`0086`](0086-shard-lineage-derived-from-extent.md). One sentence of the +Consequences below no longer holds: that a token addition under `ring` divides a token range as a +consequence of placement, which `SPLIT-001` already distinguished from a split. The distinction is +real about who decides and false about what the coordinator has to do, and its absence left `ring` +with no way to name the parent of a shard that appeared. `0086` carries that behaviour now, under +the `LIN` prefix. The withdrawal of the `range` strategy, of the `RANGE` and `SPLIT` prefixes, and +of `PROP-023`, `PROP-026`, and `CFG-063` is untouched, and `range` does not return. + ## Context [`0002`](0002-placement-strategy-set.md) shipped five strategy kinds, and justified `range` as the diff --git a/docs/design/adr/0086-shard-lineage-derived-from-extent.md b/docs/design/adr/0086-shard-lineage-derived-from-extent.md new file mode 100644 index 0000000..1ebbe98 --- /dev/null +++ b/docs/design/adr/0086-shard-lineage-derived-from-extent.md @@ -0,0 +1,104 @@ +# 0086. Shard lineage derived from extent + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0054`](0054-range-strategy-withdrawal.md) withdrew the `range` strategy and with it the whole +`SPLIT` prefix, on the ground that `SPLIT-001` admitted a split only under `range`. One sentence of +its Consequences does not hold. It reads that a token addition under `ring` divides a token range as +a consequence of placement, which `SPLIT-001` already distinguished from a split. The distinction is +real about who decides and false about what a coordinator has to do. A division that is a +consequence of placement still moves contents from one extent into another, and the machinery that +named the parent went out with the prefix. + +What that costs is visible in the port. Two `ring` snapshots at `factor` 2, with nodes `a` at tokens +`...1000` and `...5000` and `b` at `...3000` and `...7000`, gain a node `c` at the single token +`...2000`. The later snapshot enumerates a shard `...2000` that the earlier one does not, and the +contents of that shard are held by the replicas of `...3000` under the earlier snapshot. The +ownership delta reports that `...2000` appeared and that two nodes gained it. It cannot report where +the contents are, because `...3000` did not change owner and therefore has no delta entry at all. +The plan the port builds from that delta emits two handoffs whose source and destination are the +same node, and names neither node that holds the data. + +The reverse case is worse for being quieter. Removing `c` folds `...2000` back into `...3000`, whose +replica set is unchanged, so the shard that must receive the contents appears in no delta entry and +receives no handoff. + +This is reachable whenever a node joins or leaves a `ring` topology, which is the central operation +of the storage walkthrough in [`../00-overview.md`](../00-overview.md). It is not a missing feature +so much as a hole that split and merge are the names for. + +`TOPO-211` answers which shards changed owner. Nothing in the design answers which shards changed +contents, and the two questions have different answers exactly when an epoch changes which shards +exist. + +## Decision + +A **lineage** is added: the correspondence between the extents of two snapshots, where the extent of +a shard is the set of routing keys `shardOf` maps to it. Lineage relates extents and never +identifiers, which is [`0022`](0022-range-split-lineage.md)'s central insight carried forward +without the machinery that was specific to `range`. + +The lineage is derived from the topology document. No document member records it, no new strategy +kind is introduced, `formatVersion` stays at 1.0, and the JSON Schema is untouched. `0022`'s +Alternatives section rejected explicit lineage members on three grounds that all still hold: they +duplicate what the bounds, or here the tokens and the matchers, already carry; a hand-edited +document would omit them; and two members that can disagree need a rule for which one wins. + +The new requirements take the `LIN` prefix, in the Topology change and rebalancing section, on the +`migration` surface. `SPLIT` stays withdrawn and admits no further identifier, under +[`0053`](0053-requirement-withdrawal-convention.md). + +Lineage sits beside the ownership delta rather than inside it. `TOPO-211`, `TOPO-213`, and the +`ShardChange` shape are unchanged, and no `TOPO-*` requirement moves surface. + +## Consequences + +An epoch that divides or folds an extent is now plannable. A handoff for a divided extent names a +source drawn from the parent's replica set, and a fold produces a handoff for the shard that +receives the contents even where its replica set did not change, which `LIN-044` states because a +plan built over the delta alone omits it. + +The property the whole change rests on is that where two snapshots enumerate the same shard set the +lineage is the identity. Under every strategy the shard identifier of `PLACE-031` renders the +geometry that fixes the extent, so an equal shard set is an equal extent set. Every pair the suite +carried before this change is such a pair, so the regenerated tree is additions only, and a changed +expected value anywhere would mean the identity case had leaked. + +Lineage is kept out of the `routing` surface deliberately. Adding a parent to `ShardChange` would +put extent arithmetic at the `core` level, which [`../30-conformance.md`](../30-conformance.md) +says a port cannot decline, and would oblige every future port to implement ring interval +containment and directory precedence before declaring anything at all, for behaviour an integrator +routing tenant identifiers never uses. + +`range` does not return. Nothing here reinstates an ordered keyspace, a scan, or the `RANGE` prefix, +and a further strategy kind remains the minor format version that +[`../99-roadmap.md`](../99-roadmap.md) prices. + +## Alternatives + +Reinstating a range-like ordered strategy. Rejected. It costs a minor format version, a fifth entry +in a closed set, a fresh prefix of some forty identifiers, and the `assignment` default trap +[`0043`](0043-assignment-mode-defaults.md) chose to state loudly rather than remove, and at the end +of it split would be available only under the one kind that none of the three validated use cases +reaches. The `ring` hole would still be open. + +Making `slot` splittable by doubling `slotCount`. Rejected, and recorded here because the arithmetic +is more encouraging than the conclusion and should not be rediscovered. Writing a key hash as +`kn + i`, its remainder modulo `2n` is `i` where `k` is even and `i + n` where `k` is odd, so +doubling sends every key of slot `i` to slot `i` or slot `i + n` and nowhere else, for any `n`, with +no power-of-two requirement and no bitmask. A doubling is therefore a clean binary refinement of +every slot at once and a halving is its fold. It is still not worth building: under `slot` the +extent of a shard never changes while `slotCount` is held equal by `TOPO-231`, the delta and the +plan already work, and doubling addresses only a `slotCount` chosen too small, which an authority +avoids by choosing generously. + +A lineage layer above the strategies, with parent and child recorded per shard. Rejected for +`0022`'s reasons, restated above. + +Deriving a lineage by sampling routing keys and observing which shard each falls in. Rejected. It is +unsound at any sample size, and it would make a lineage depend on something outside the two +documents, which `LIN-005` forbids for the same reason `TOPO-221` forbids it of the delta. diff --git a/docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md b/docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md new file mode 100644 index 0000000..2133399 --- /dev/null +++ b/docs/design/adr/0087-lineage-capability-derived-rather-than-declared.md @@ -0,0 +1,86 @@ +# 0087. Lineage capability derived rather than declared + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0086`](0086-shard-lineage-derived-from-extent.md) adds a lineage. A question it does not settle is +how an implementation knows whether a lineage is available for a given pair of snapshots, and the +obvious answer is wrong in a way worth recording. + +The obvious answer is a capability method on the placement extension point, defaulted to answer that +no lineage is offered. It reads as though it follows `MOVE-261`, and it does not. `MOVE-241` derives +support for orchestrated migration from observable geometry: a strategy supports it exactly when +`shards()` is non-empty and `shardOf` returns a shard identifier rather than the routing key. +`MOVE-261` binds a registered strategy to that same geometric rule and forbids inferring support +from the strategy's name. The design derives capability; a defaulted capability method declares it. + +The difference is not stylistic. [`../30-conformance.md`](../30-conformance.md) says a port declares +the levels it reaches, not a percentage. A port that declared the `migration` level while answering +that it offers no lineage for `ring` would be indistinguishable from a port that had simply not done +the work, and the configuration that produces is exactly the one that emits the self-handoffs +`0086` describes. + +Underneath the mistake were two different questions fused into one flag. + +## Decision + +The two questions are separated, and each is answered in its own way. + +Whether a **strategy** has a lineage is derived and never declared. A strategy has one exactly when +it supports orchestrated migration under `MOVE-241` and its shard extents are determined by the +topology document. `ring`, `slot`, and `directory` have one; `rendezvous` does not, because it +enumerates no shard. There is no opt-out, because the extents exist whether or not an implementation +has computed them. Lineage support is therefore not a property of the design at all, it is a +property of an implementation's completeness, and completeness is what the level system exists to +police. + +Whether the **integrator's storage** can divide a shard in place is declared, in +`HookDeclaration`. That is a statement about a system the library cannot observe, and +[`0085`](0085-hook-declarations-and-refused-aborts.md) is the precedent: `supportsRollback` and +`supportsVerify` are read as the integrator's statement about their own system, and the library +refuses rather than calling a hook that was never implemented. Nothing about it is a statement about +a port's completeness, so it does not collide with the level system. + +A registered strategy outside the core set is the one case where the library cannot derive the +answer, and it supplies its lineage rather than declaring a capability. `LIN-015` therefore forbids +deriving a registered strategy's extents by sampling and forbids inferring a lineage from the +strategy's name, and where a registered strategy supplies none and the two snapshots enumerate +different shard sets, the plan is refused with the cause `lineageUnsupported`. The method survives +but its meaning changes: it does not say "I support lineage", it says "here is my lineage", and its +absence has a stated, observable consequence rather than a silent one. + +## Consequences + +Per-strategy scoping does not need a new mechanism, because the suite already has one. The strategy +surface axis of [`../30-conformance.md`](../30-conformance.md) selects vector files on the surfaces +a port exposes, and `run_suite.py --strategy` already drives it. A port that exposes `rendezvous` +and `directory` and declares `migration` implements directory lineage and no ring lineage, and that +is expressible today. + +Staging within this repository is done with a stated refusal rather than a silent absence. +`LIN-013` refuses a `directory` pair whose shard sets differ, with the cause `unalignedLineage`, +until directory extents are defined; a later change restates that requirement and removes the +refusal, keeping its identifier under [`0053`](0053-requirement-withdrawal-convention.md). A refusal +is a conforming, testable behaviour that a vector pins, so no port can be green on a hole. + +This improves `directory` immediately rather than deferring it. A prefix refinement today produces +the same silent self-handoff as a ring token addition. Under the staged refusal it produces a +refused plan naming a cause, which is a correct answer in place of a wrong one. + +## Alternatives + +A defaulted capability method meaning "I offer no lineage". Rejected above: it makes an incomplete +port indistinguishable from a complete one, against the rule that a port declares levels rather than +a percentage. + +A per-strategy capability flag in the conformance declaration. Rejected. It duplicates the strategy +surface axis that already exists, and it would let a port declare `migration` and `ring` together +while opting out of the one thing that makes the pair meaningful. + +Recording directory lineage under "Requirements without an executable test" so it could ship before +its vectors. Rejected. A directory extent is decidable over a prefix trie, so it is testable, and +recording something as untestable to avoid writing its vectors is the one dishonest move available +here. diff --git a/docs/design/adr/0088-lineage-inside-the-migration-surface.md b/docs/design/adr/0088-lineage-inside-the-migration-surface.md new file mode 100644 index 0000000..b0fcc58 --- /dev/null +++ b/docs/design/adr/0088-lineage-inside-the-migration-surface.md @@ -0,0 +1,54 @@ +# 0088. Lineage inside the migration surface + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0086`](0086-shard-lineage-derived-from-extent.md) adds the `LIN` prefix. Where its requirements +are tested, and which implementations they bind, is a separate judgement, because +[`0058`](0058-conformance-surfaces.md) makes a surface a unit an implementation exposes whole or not +at all and [`0052`](0052-conformance-level-partition.md) fixes the partition of the suite into +levels. + +Three placements were available: a new `lineage` surface with a level of its own, the existing +`migration` surface and level, or the `routing` surface alongside the ownership delta. + +## Decision + +Lineage belongs to the existing `migration` surface and is tested at the existing `migration` level. +No surface is added and the level partition does not move. + +## Consequences + +A port that declines the `migration` surface implements no lineage and is bound by no `LIN` +requirement, which is the same answer `0058` already gives for every other part of the handoff +coordinator. + +A port that exposes `migration` implements lineage for every strategy surface it also exposes. There +is no configuration in which a port exposes the handoff coordinator over `ring` and has no answer +for a token addition, because that configuration is the one that emits the self-handoffs `0086` +describes. + +The level partition is untouched, symmetrically with `0054`'s own argument that removing artefacts +from a level does not move the partition. `migration` already requires `failover`, and it gains +vector files rather than a new requires edge. + +The Java port declares every level, so it pays for lineage at `migration` and for the `TOPO-213` +repair at `core`. No other port exists. A future port declining `migration` pays nothing, and a +future port exposing only `rendezvous` pays nothing beyond the `core` repair, because `rendezvous` +enumerates no shard and `MOVE-241` already excludes it. + +## Alternatives + +A `lineage` surface of its own. Rejected because it would let a port declare `migration` without it, +which is the broken configuration named above. + +Lineage on the `routing` surface, as a parent member of `ShardChange`. Rejected in `0086`'s +Consequences: it would put ring interval containment and directory precedence at the `core` level, +which a port cannot decline, for behaviour an integrator routing tenant identifiers never uses. + +A new conformance level. Rejected. A level tests one surface, `migration` is that surface, and a +second level over the same surface would let a port claim the coordinator while declining the part +that makes it correct. diff --git a/docs/design/adr/0089-a-division-undone-by-a-combination.md b/docs/design/adr/0089-a-division-undone-by-a-combination.md new file mode 100644 index 0000000..f50208c --- /dev/null +++ b/docs/design/adr/0089-a-division-undone-by-a-combination.md @@ -0,0 +1,71 @@ +# 0089. A division undone by a combination + +Date: 2026-09-19 + +Status: accepted + +## Context + +[`0086`](0086-shard-lineage-derived-from-extent.md) makes a division plannable, and a division that +happens entirely on one node is the part with no precedent in the state machine. A node that holds +the parent under the earlier snapshot and the child under the later one moves nothing to anybody. It +has to divide its own copy so that what it holds matches the extent it now owns. + +[`0022`](0022-range-split-lineage.md) had this case and answered it in a way +[`0054`](0054-range-strategy-withdrawal.md) was glad to be rid of. `SPLIT-171` required a handoff +whose local split had already succeeded to move to `failed(undetermined)` without a re-observation, +because the data no longer matched the parent's bounds and could not be put back. That was the +single carve-out in `MOVE-233`, and `0054` recorded its removal as a gain: +[`0051`](0051-recovery-from-an-undetermined-cutover.md) wanted `MOVE-233` unconditional, and with +`SPLIT` withdrawn it became so. + +Reinstating the local step reopens the question. If a succeeded division cannot be undone, the +carve-out comes back with it. + +## Decision + +A division is reversible, and the inverse of a division is a combination. + +`LIN-056` requires an aborted local step to be undone by the inverse hook: a division is undone by +`combine`, a fold by `divide`. A local step therefore reaches `aborting` like every other state that +does, `MOVE-421` gains `dividing` beside the three states it already names, and `MOVE-233` keeps the +unconditional form `0051` wanted. There is no carve-out. + +The cost lands on the integrator, visibly. `LIN-053` refuses a plan that requires a local step where +`declare` answers `supportsLineage` of false, which is the shape +[`0085`](0085-hook-declarations-and-refused-aborts.md) established: a declaration is the +integrator's statement about their own storage, and the library refuses rather than calling a hook +that was never implemented. + +A handoff whose inverse fails and whose attempts are exhausted reaches `failed` with the new kind +`undivided`. The four kinds `MOVE-011` already carries each name a condition of a copy that moved +between nodes, and this one never left the node it is on. + +## Consequences + +An integrator whose storage can divide a shard but not recombine two adjacent ones declares +`supportsLineage` of false and gets a refused plan rather than a handoff that can only fail. That is +a real restriction, and it is the honest one: a step that cannot be undone is not an abortable step, +and pretending otherwise is what produced `SPLIT-171`. + +The state machine gains one state and four transitions. `MOVE-021` says "exactly these transitions +and no others", so this is the largest single edit in the batch, and it lands after the lineage +computation is settled and vector-backed rather than beside it. + +A division whose children keep the parent's replica set is a local step alone, with no data crossing +the network, which is `0022`'s own observation and still true. + +## Alternatives + +A succeeded local step that cannot be undone, as `SPLIT-171` had it. Rejected. It reinstates the +carve-out in `MOVE-233` that `0051` argued against and `0054` removed, and it makes one state in the +machine behave unlike every other. + +Requiring no local hook at all, and asking the integrator to divide its copy out of band. Rejected. +The library would have no way to know whether the division had happened, so it could not sequence a +handoff that draws from the divided parent, which `LIN-057` has to order. + +Treating a local step as a handoff from the node to itself with the ordinary hook sequence. +Rejected. `prepare`, `transfer`, `catchUp`, `quiesce`, and `commitCutover` all describe two parties, +and a single-winner cutover between a node and itself is meaningless. The shorter path through +`dividing` says what actually happens. diff --git a/docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md b/docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md new file mode 100644 index 0000000..e35c1f3 --- /dev/null +++ b/docs/design/adr/0090-no-rebase-across-a-lineage-boundary.md @@ -0,0 +1,49 @@ +# 0090. No rebase across a lineage boundary + +Date: 2026-09-19 + +Status: accepted + +## Context + +`MOVE-096` makes a handoff rebasable onto a newer snapshot exactly when that snapshot enumerates the +handoff's shard, its source, and its destination. +[`0050`](0050-plan-rebase-onto-a-newer-snapshot.md) settled that rule before a lineage existed, when +the only way a shard could leave a snapshot was for the authority to stop naming it. + +With [`0086`](0086-shard-lineage-derived-from-extent.md) a shard can leave a snapshot because its +extent was divided again. A handoff moving contents into a shard that a third epoch has since +divided has a shard the newer snapshot does not enumerate, so `MOVE-096` drops it. + +The question is whether the rebase should instead follow the lineage: recognise that the shard's +extent still exists under new names, and rewrite the handoff onto the children. + +## Decision + +It should not. A handoff whose shard the newer snapshot does not enumerate is aborted and +compensated, exactly as `MOVE-096` already has it. No `MOVE-09*` requirement changes. + +## Consequences + +A split overtaken mid-flight by a further split of the same region costs the work already done. The +destination's partial copy is released by `rollback` and the newer plan starts the move again, which +is correct but not free. + +The behaviour is what the port already does, so nothing in the coordinator changes and no scenario +moves. + +Relaxing a refusal later is cheap and withdrawing a clever rebase that proved wrong is not. A +transitive rebase would have to decide which child inherits a partially transferred copy, whether a +quiesce lease taken against the parent still binds the children, and what a cutover record naming +the parent means once the parent is gone. None of those has an obvious answer, and getting one wrong +loses data rather than time. + +## Alternatives + +Rewriting a handoff onto the children of its shard. Rejected above: it is a larger change than it +looks, it is not reversible once shipped, and the case it optimises is an authority publishing two +divisions of one region in quick succession, which is rare and already correct if slower. + +Refusing the rebase outright rather than aborting the handoff. Rejected. `MOVE-096` classifies each +handoff independently and a plan may contain many, so refusing the whole rebase because one handoff +cannot be carried would abandon the ones that can. diff --git a/docs/design/adr/0091-lineage-classification-as-a-signal.md b/docs/design/adr/0091-lineage-classification-as-a-signal.md new file mode 100644 index 0000000..02d0245 --- /dev/null +++ b/docs/design/adr/0091-lineage-classification-as-a-signal.md @@ -0,0 +1,65 @@ +# 0091. Lineage classification as a signal + +Date: 2026-09-19 + +Status: accepted + +## Context + +The motivation for [`0086`](0086-shard-lineage-derived-from-extent.md) was that splitting and +merging should be the ordinary way to change capacity, in preference to republishing a topology that +redistributes everything. That raises a question the lineage does not answer by itself: whether the +library should do anything about an authority that reshards wholesale when a split would have done. + +Three positions were available. Refuse the plan where the lineage shows a cheaper change existed. +Emit a warning. Do neither, and let the classification be reported. + +The library authors no topology document, assigns no epoch, and edits nothing. +[`0022`](0022-range-split-lineage.md) was firm that the decision to divide belongs to the authority, +and nothing in `0086` changes that. + +## Decision + +The library refuses `unaligned` and refuses nothing else. + +`LIN-022` refuses a plan over a lineage that classifies any shard `unaligned`, because a boundary +that moves without either dividing or folding an extent whole has no correspondence an +implementation can name. That is a correctness refusal over an ambiguous input, not a policy one, +and the requirement states the remedy, which is `0022`'s and still the right one: publish the change +as two epochs, the first dividing every extent the change crosses and the second folding the pieces +into their destinations. + +The classification is reported through the `migration.lineage` event of `OBS-020`, which carries the +count per class. The planner needs the classification to choose a source, so reporting it costs +nothing. + +No refusal and no warning is raised for a change that could have been expressed as a division. The +preference is carried by documentation, which states when a division is the right shape, when a +wholesale republication is, and what each costs. + +## Consequences + +An operator reads from `migration.lineage` whether an epoch was a refinement or a redistribution, +and decides what to do about it. The library states the facts and takes no position. + +The strongest expression of the preference is not a rule at all. Before `0086` an authority that +added one node to a ring got a plan of self-handoffs, and an authority that republished everything +got the same, so nobody preferred anything because neither worked. Making a division correct and +cheap is most of what "preferred" can honestly mean here. + +An authority that reshards wholesale where a division would have served pays for it in movement and +is told nothing by the library. That is the accepted cost of not guessing. + +## Alternatives + +Refusing a plan where the lineage shows a division was available. Rejected. The library cannot +distinguish a deliberate rebalance, a seed rotation, or a failure domain re-layout from a lazy +reshard, and a false refusal blocks an epoch the authority has already published to routing, which +is a worse failure than the one it would prevent. + +A warning event for the same condition. Rejected for the same reason at lower value: it needs a +threshold for what counts as wholesale, which is a judgement the library has no basis to make, and +an operator who cannot act on it learns to ignore it. + +A configurable policy selecting among the three. Rejected. It is a knob for a policy nobody has +asked for, and it would oblige every port to implement all three positions. diff --git a/docs/design/adr/0092-an-observability-sink-for-the-coordinator.md b/docs/design/adr/0092-an-observability-sink-for-the-coordinator.md new file mode 100644 index 0000000..1776806 --- /dev/null +++ b/docs/design/adr/0092-an-observability-sink-for-the-coordinator.md @@ -0,0 +1,70 @@ +# 0092. An observability sink for the coordinator + +Date: 2026-09-19 + +Status: accepted + +## Context + +`OBS-020` tables ten events under `migration.`, and an implementation emits the events of the +surfaces it exposes. The Java port declared all ten in its inventory, which is what the +`migration` inventory vector reads, and emitted none of them. Nothing detected that, because no +driver asserts an emitted event for any surface. + +The reason the port emitted none is structural rather than an oversight at a call site. A router is +built from a `RouterConfig`, which carries the metrics registry and the event sink, so +`DefaultRouter` holds a `MetricsHolder` and emits the `routing` events through it. A coordinator is +built by `Sharder.coordinator()`, which takes nothing, so there is no sink to emit through and +nowhere for one to come from. + +The coordinator is deliberately stateless between calls under `MOVE-061`, and a plan is a pure +function of the two snapshots and the policy under `MOVE-221`. Neither of those is in tension with +reporting what happened, but both rule out the coordinator acquiring a sink by holding onto a +router. + +## Decision + +`Sharder.coordinator(RouterConfig)` is added beside `Sharder.coordinator()`. The configuration +carries the metrics registry and the event sink the router already reads, and the coordinator reads +the same two and nothing else from it. + +`Sharder.coordinator()` stays, and means a coordinator that reports nothing. It is the right call +for an integrator that has not configured observability, and removing it would make the simple case +carry a configuration it has no other use for. + +`MigrationPolicy` does not carry the sink. It is a record of tuning values that a plan reports as a +payload member of `migration.planned`, and putting a dependency inside it would make two plans +incomparable that differ only in where their events go. + +`plan` and `lineage` do not take a sink per call. A sink is a property of the process rather than of +one migration, and threading it through every call would put it in the signature of every future +operation. + +## Consequences + +The precedent is `Sharder.loader(RouterConfig)`, which already takes the whole configuration to +answer a narrower question. An integrator that has a `RouterConfig` for its router passes the same +one, and the coordinator reads the two observability members from it. + +A coordinator built from a configuration whose observability is unset behaves exactly as one built +from none, because `MetricsHolder` already treats an absent registry and an absent sink as counting +by name and discarding. + +The events a running coordinator emits are still not asserted by the suite, for any surface. This +record does not change that, and +[`../30-conformance.md`](../30-conformance.md#observability-vectors) now says so plainly rather than +claiming an assertion that no driver performs. + +## Alternatives + +A sink on `MigrationPolicy`. Rejected above: the policy is data a plan reports, not a dependency. + +A sink parameter on `plan` and `lineage`. Rejected: it is per process, not per call. + +A coordinator that takes a `Router` and reads its sink. Rejected. It would tie the coordinator's +lifetime to a router's and give it a snapshot it must not read, when what it needs is two members of +a configuration. + +Leaving the coordinator silent and recording the ten events as unimplementable. Rejected. They are +implementable, the contract is published in `OBS-020`, and the withdrawal register makes each name +permanent, so a port that emits nothing has broken a contract rather than deferred one. diff --git a/ports/java/USER-GUIDE.md b/ports/java/USER-GUIDE.md index 178ca8c..f36c4fb 100644 --- a/ports/java/USER-GUIDE.md +++ b/ports/java/USER-GUIDE.md @@ -1,9 +1,9 @@ # Java port user guide -A task-oriented walk through the Java port of the sharder library: a first routed key, the node set -it routes over, replication, failover, read affinity, fencing, resharding, and orchestrated -migration. Each section assumes the ones above it, and each ends at the reference that carries the -full detail. +A task-oriented walk through the Java port of the sharder library: how a control plane and a caller +divide the work, a first routed key, the node set it routes over, replication, failover, read +affinity, fencing, resharding, and orchestrated migration. Each section assumes the ones above it, +and each ends at the reference that carries the full detail. ## Scope of the library @@ -29,6 +29,117 @@ function of the snapshot in force, the routing key, and the signals reported. A caller is the application embedding the library. It is not the client that talks to a node; that one belongs to the integrator's world. +## The control plane and the caller + +### Division of labour + +A topology authority decides what the cluster looks like and publishes a topology document for each +epoch. The authority sits outside the library: a control plane, a key-value store, a file under +configuration management, or an operator with an editor. The library assigns no epoch, elects no +authority, and writes no document. + +Each process that routes embeds the library and holds a `Router` over the document the authority +published. A router is long-lived. It is constructed once, it holds the snapshot in force, and it is +closed at shutdown; one constructed per request would repeat the preparation a snapshot pays for +once. + +| Lifetime | What is held | +|---|---| +| the process | one `Router`, and the `TopologyProvider` it reads documents from | +| an epoch | the immutable `TopologySnapshot` the router installed | +| one request | the `RoutingDecision`, the `AttemptSequence` over it, and its `FencingToken` | + +The authority publishes and the caller routes, and at run time neither asks anything of the other. +Two callers holding the same topology identifier and epoch compute the same preference list for the +same key, so no caller is told which node to use and none asks. +[`00-overview.md`](../../docs/design/00-overview.md#component-model) draws the components and the +three extension points an integrator implements. + +### A document reaching a running router + +A topology document is JSON, and where it comes from is the integrator's: a file on disk, a +key-value store, or octets a control plane already holds. It reaches a router through a +`TopologyProvider`, which delivers those octets and never constructs a snapshot. Validation, the +canonical form, the digest, the epoch comparison, and the preparation are the library's. +[Topology providers](#topology-providers) gives the interface, and the two providers the library +carries are the sections below it. + +An epoch is published and never inferred. The library assigns none and increments none. It compares +a candidate document against the snapshot in force by integer comparison of `epoch` and octet +comparison of `topologyId`, reading no clock, no provider revision, and no document timestamp. A +document below the epoch in force is refused as stale, one at that epoch carrying the same digest is +a no-op that refreshes freshness, one at that epoch carrying a different digest is refused as a +conflict, and one above it is installed. +Two settings are checked before any of that, and both hold when nothing is in force, which is the +state a process is in after a restart. `expectedTopologyId` names the cluster this process routes +for, so a document under any other identifier is refused as a conflict rather than adopted, which is +what an unconfigured router does with the first document it accepts. `minEpoch` is a floor, and a +document below it is refused as stale even where it is the first to arrive, which is how an operator +stops a process coming back up on a document older than the one it was serving. + +```java +RouterConfig config = RouterConfig.builder() + .provider(provider) + .expectedTopologyId("orders") + .minEpoch(41) + .build(); +``` + +[`10-specification.md`](../../docs/design/10-specification.md#monotonicity-and-acceptance) carries +the whole acceptance table, including the order the rows are evaluated in: the identifier is checked +before the floor, so a foreign document below the floor is a conflict and not stale. + +Installation replaces the snapshot in force whole, as a single atomic replacement of the reference a +routing call reads. A routing call already under way read that reference at entry and computes its +whole result from the snapshot it read, so an installation changes no call in flight and invalidates +no decision already answered. The router retains a bounded number of earlier snapshots for +[the recipient check](#the-recipient-check), and routes against none of them. + +### The nodes for one key + +The everyday call takes a key and answers the nodes that hold it, in the order to ask them. + +```java +RoutingDecision decision = router.route("tenant-42", RouteOptions.DEFAULTS); +NodeId owner = decision.primary().node(); +List candidates = decision.entries(); +FencingToken token = decision.token(); +``` + +`primary()` is the head of the list and the node to ask first. `entries()` holds that head and the +nodes behind it, each entry carrying its position, its `REPLICA` or `FALLBACK` role, its health +state, and whether the health filter left it attemptable. `shard()` names the shard the key belongs +to under the strategies that enumerate shards, and `token()` is what a request carries so the +recipient can check it. [The routing decision](#the-routing-decision) gives every member. + +Trying the nodes in turn is `router.attempts(decision)` rather than a loop over `entries()`, because +the walk applies the health filter, the attempt limit, and the retry budget. +[Walking an attempt sequence](#walking-an-attempt-sequence) gives it. + +The cost is bounded, and where it is paid matters more than what it is. + +- A routing call is a pure function of the snapshot in force, the routing key, and the health + signals the caller reported. It reads the snapshot reference once at entry and takes no lock. +- The work that grows with the node set is preparation, and preparation is performed once per + snapshot at installation rather than once per call. + [`10-specification.md`](../../docs/design/10-specification.md#placement-cost-model) bounds + preparation, one routing call, and the resident size of a prepared placement. +- A call consumes a bounded prefix of the candidate ordering rather than the whole ordering: the + greater of the achieved replica count and the resolved attempt limit, raised by the entries the + health filter skips. `CORE-046` fixes the prefix and `PLACE-071` bounds it, and + [`adr/0034`](../../docs/design/adr/0034-lazy-candidate-traversal-surface.md) records the cursor + that leaves the ordering beyond the prefix uncomputed. +- `rendezvous` is the exception. It determines its first candidate from the scores of the whole + eligible node set, so no prefix of its ordering costs less than the whole; `ring`, `slot`, and + `directory` pay for the prefix alone. +- `preferenceList()` and an explain record each consume the whole ordering, so both belong off the + routing path. + +The Java build holds an allocation gate over one routing call, in +`src/test/java/com/codeheadsystems/sharder/api/AllocationGateTest.java`. A ring of a hundred nodes +and a ring of a thousand take one ceiling between them, and a rendezvous topology takes a ceiling of +its own. + ## A first router ### Dependency coordinates @@ -533,27 +644,84 @@ topology before retrying, and never a reason to install that epoch. [`10-specification.md`](../../docs/design/10-specification.md#caller-behaviour-when-fenced) states the walk. -## Resharding +## Resharding and shard lineage + +An authority changes a topology by publishing a whole document at a higher epoch. What that change +does to the shards is the thing worth being deliberate about, and there are two shapes. + +A **reshard** keeps the shards and moves their ownership. The node set or the replication factor +changes, every shard keeps its identifier and the keys it holds, and the ownership delta names the +shards whose replica set differs. + +A **split** or a **merge** changes which shards exist. Adding a ring token divides the extent of one +shard into two; removing one folds two extents into one. The shard identifiers are not the same on +both sides, so the ownership delta, which joins on the identifier, has no answer for the ones that +appeared or vanished. The lineage is the second join, over the keys a shard holds, and it is what +tells a plan where a new shard's contents come from. -The library has no split operation and no merge operation. -[`adr/0054`](../../docs/design/adr/0054-range-strategy-withdrawal.md) withdrew the `range` strategy -and the split prefix whole. Resharding takes a different shape: an authority publishes a topology at -a higher epoch with a different node set or a different replication factor, ownership of the -existing shards moves, and the delta between the two topologies names exactly what moved. +The library derives the lineage from the topology document. Nothing in the document records it, and +no member has to be authored to get a split. + +### Which shape to reach for + +Prefer a split or a merge where the change is about capacity for part of the keyspace. + +| | Split or merge | Reshard | +|---|---|---| +| What changes | which shards exist, and the keys each holds | who owns the existing shards | +| What moves | the contents of the extents that divided or folded | the contents of every shard whose replica set changed | +| Typical cause | one shard outgrew a node, or two are small enough to combine | a node joined or left, or the replication factor changed | +| Cost | proportional to the extent that moved | proportional to what the strategy reassigns | +| Reversible | yes, by folding back or dividing again | yes, by publishing the earlier shape | + +A split moves less because it disturbs less: the keys outside the divided extent do not change +shard, so nothing about them moves. A reshard that redistributes the whole keyspace to add capacity +moves data that was already where it belonged. Where both would serve, the split is the cheaper +change, and it is the one to publish. + +A reshard is the right shape when what changed is the cluster rather than the keyspace. Adding a +node to a `ring` topology under derived tokens is a reshard and a split at once, because the node's +tokens divide the extents they land in, and the library plans it as one change. + +Under `slot` the shard count is fixed by `slotCount`, which `TOPO-231` holds equal across a +comparable pair, so every change is a reshard. Choose `slotCount` generously at the outset: it is +the one decision here that a later epoch cannot revisit. + +The library does not enforce the preference. It refuses a change whose boundaries neither divide nor +fold an extent whole, because that has no lineage to name, and it reports the classification through +the `migration.lineage` event so an operator can see which shape an epoch took. It refuses nothing +else, and it never declines a reshard on the ground that a split would have been cheaper, because it +cannot tell a deliberate rebalance from a lazy one. +[`adr/0091`](../../docs/design/adr/0091-lineage-classification-as-a-signal.md) argues that. + +### The unaligned change + +A change that moves a boundary without either dividing or folding an extent whole is refused with +`PlanRefusedException` and the cause `unalignedLineage`. Removing a ring token and adding a +different one inside the same extent in one epoch is the common way to reach it. + +Publish it as two epochs instead: the first divides every extent the change crosses, the second +folds the pieces into their destinations. Each is plannable on its own, and each leaves a topology +that routes correctly if the second is delayed. + +Under `directory` an extent is a matcher narrowed by the entries that outrank it, so refining +`prefix:ab` into `ab0` and `ab1` is a division and dropping the two back to `ab` is a fold. An entry +that wins keys the earlier table matched to nothing names a shard with no parent, which moves no +contents and needs no handoff. ### A new epoch -An epoch is published, never inferred. The library assigns no epoch, increments none, and accepts no -document whose epoch is at or below the one in force under the same identifier. A change is a whole -document at the next epoch. +A change is a whole document at the next epoch, published the way every other document reaches a +router. ```java provider.publish(documentAtEpochTwo); ``` -The snapshot in force is replaced whole at installation. A routing call under way reads the snapshot -it started against from first candidate to last, so an installation changes no decision already -taken. +The epoch belongs to the authority, and what the library does with an arriving document is under +[A document reaching a running router](#a-document-reaching-a-running-router). A document below the +epoch in force is refused, so an epoch that carried a split is not undone by republishing the epoch +before it; the way back is a further epoch that folds the pieces again. ### The ownership delta @@ -577,17 +745,86 @@ A `ShardChange` carries the shard, the replica set before, the replica set after differences. The sets are the entries whose role is `REPLICA`, which is the achieved replica prefix rather than the whole preference list, so a node a shard merely falls back to has gained nothing. -The delta is computed on demand rather than at installation, and a shard the later snapshot does not -enumerate is absent from it. Two snapshots whose shard identity is not comparable refuse the -computation with `InvalidArgumentException` rather than reporting every shard as wholly changed. +The delta is computed on demand rather than at installation. Its entries come in two groups: first +the shards the later snapshot enumerates, in that snapshot's order, then the shards only the earlier +one enumerates, so a shard that vanished reports the nodes that lost it rather than going +unreported. Two snapshots whose shard identity is not comparable refuse the computation with +`InvalidArgumentException` rather than reporting every shard as wholly changed. The same pair given +to `plan` is refused with `PlanRefusedException` and the cause `incomparableShards`, so one +condition reaches a caller as whichever condition the surface it called raises. Under `rendezvous` the delta is empty, because the strategy enumerates no shards. Reading the delta is enough where the integrator's store moves data by itself. Where each move has -to be sequenced so that no request is served by both the old owner and the new one, the coordinator -is the next section. +to be sequenced so that no request is served by both the old owner and the new one, +[Orchestrated migration](#orchestrated-migration) gives the coordinator. [`10-specification.md`](../../docs/design/10-specification.md#ownership-delta) states the delta. +### The lineage operation + +`coordinator.lineage(from, to)` answers where each shard's contents come from across two snapshots. +It installs nothing and plans nothing, so an integrator sizes a change before deciding to run it, +which is the reason [the ownership delta](#the-ownership-delta) is a call rather than a product of +installation and the reason this is one too. + +The two answer different questions, and an epoch that changes which shards exist needs both. + +| | Ownership delta | Lineage | +|---|---|---| +| Joins on | the shard identifier | the keys a shard holds | +| Answers | which shards changed owner | where a shard's contents are | +| A shard that appeared | reports it gaining every replica | names the parents its extent draws from | +| A shard whose extent grew and kept its replicas | no entry | `MERGED`, with its parents | +| Under `rendezvous` | empty | refused | + +```java +ShardLineage lineage = coordinator.lineage(before, after); +for (ShardLineage.Entry entry : lineage.entries()) { + ShardId shard = entry.shard(); + LineageClass became = entry.lineage(); + List parents = entry.parents(); +} +``` + +`LineageClass` holds eight values. `UNCHANGED` and `MOVED` are the two whose extent is the one the +parent held, which `extentUnchanged()` reports, and which every shard falls into where an epoch +moved ownership alone. + +| Class | What became of the shard | +|---|---| +| `UNCHANGED` | one parent of equal extent, and the replica set is equal | +| `MOVED` | one parent of equal extent, and the replica set differs | +| `DIVIDED` | one parent whose extent strictly contains this one | +| `MERGED` | two or more parents whose extents this one contains | +| `FRESH` | no parent, this extent meeting no extent of the earlier snapshot | +| `SPLIT` | only the earlier snapshot enumerates it, and its extent divides into two or more | +| `FOLDED` | only the earlier snapshot enumerates it, and one child's extent contains it | +| `VACATED` | only the earlier snapshot enumerates it, and its extent meets no later extent | + +`parents()` names the shards of the earlier snapshot for an entry the later snapshot enumerates, and +the shards of the later one for an entry only the earlier snapshot enumerates. It is empty under +`FRESH` and `VACATED`. The entries come in the two groups the ownership delta comes in: first the +shards the later snapshot enumerates, in that snapshot's order, then the shards only the earlier one +enumerates. + +`lineage.entry(shard)` answers one entry. `lineage.counts()` answers how many shards fall in each +class, which is what the `migration.lineage` event reports. `lineage.identity()` answers whether +every shard kept the extent it held, which is every pair under `slot` and every pair under any kind +whose shard set did not move. + +The call refuses with `PlanRefusedException`, whose `reason()` carries one of three causes. Two +snapshots carrying different topology identifiers, and two whose shard identity is not comparable, +both give `incomparableShards`, because a pair under two identifiers joins on nothing and `plan` +answers the same cause for the same input. A strategy that enumerates no shard, which is +`rendezvous`, gives `strategyUnsupported`. A boundary that moved without either dividing or folding +an extent whole gives `unalignedLineage`, which is [the unaligned change](#the-unaligned-change). + +A snapshot this library did not produce is an `InvalidArgumentException` rather than a refusal, as +it is for `plan`. + +[`10-specification.md`](../../docs/design/10-specification.md#lineage-classification) states the +classes. + ## Orchestrated migration ### Store and strategy preconditions @@ -608,13 +845,36 @@ HandoffCoordinator coordinator = Sharder.coordinator(); MigrationPlan plan = coordinator.plan(before, after, hooks, MigrationPolicy.defaults()); ``` +`Sharder.coordinator(config)` answers a coordinator that reports the `migration.` events of +`OBS-020` through the registry and the sink the configuration carries. It reads those two members +and nothing else from it: no provider, no snapshot, and no health view. +`Sharder.coordinator()` reports nothing, which is the right call where observability is unset. + The coordinator holds nothing between calls and installs nothing. A plan is a pure function of the two snapshots and the policy, which is what lets a restarted coordinator rebuild the same plan. A plan is never created as a side effect of installing a snapshot. `plan` refuses with `PlanRefusedException` where the two snapshots do not join on shard identity, -where the target epoch does not advance, where the strategy supports no orchestrated migration, or -where a destination sits outside the target placement set. +where the target epoch does not advance, where the strategy supports no orchestrated migration, +where a destination sits outside the target placement set, where the change is unaligned, or where +the change needs a local step the hooks declare no support for. `PlanRefusedException.Cause` names +each of them. + +The handoffs come from the lineage rather than from the ownership delta. One handoff is admitted for +each destination of each shard of the later snapshot that does not already hold the contents of a +parent that shard's extent draws from, so a shard whose extent grew carries a handoff even where its +replica set is unchanged and the delta reports nothing for it. A shard with no parent has no +contents to move and carries none, and no handoff names one node as both its source and its +destination. +[`10-specification.md`](../../docs/design/10-specification.md#plan-construction-over-a-lineage) +states the derivation. + +A split or a merge also needs work on a node that holds the parent under the earlier snapshot and +the child under the later one, and that work moves nothing between nodes: the node divides or folds +its own copy so that what it holds matches the extent it owns. A plan carries it as a local step, +whose source and destination are the same node, sequenced before every handoff that draws from a +divided parent and after every handoff that draws into a folded shard. A local step counts against +the policy's concurrency as a handoff does. A plan is passive. It advances when `step` is called and never on a timer, a thread, or an installation. One `step` advances at most one handoff by at most one hook call. Concurrent calls to @@ -663,28 +923,45 @@ public interface MovementHooks { VerifyResult verify(HandoffContext context); HookResult cleanup(HandoffContext context); HookResult rollback(HandoffContext context); + HookResult divide(HandoffContext context); + HookResult combine(HandoffContext context); ObserveResult observe(HandoffContext context); } ``` +`divide` and `combine` are the hooks a local step calls, and each carries a default that refuses +permanently. A storage that has not implemented them declares `supportsLineage` of false, and the +plan that would call one is refused before any call is made. + Every hook is called from whichever unit of execution called `step`, at most one per call. A hook that raises rather than answering has its condition surfaced unchanged. -`HandoffContext` tells a hook the shard, the topology identifier, the source epoch and the target -epoch, the source node, the destination node, the attempt number, and the deadline in milliseconds. -The target epoch is the handoff's own rather than the plan's, so the two differ after a rebase. +`HandoffContext` tells a hook the shard, the shard its contents come from, the topology identifier, +the source epoch and the target epoch, the source node, the destination node, the attempt number, +and the deadline in milliseconds. `sourceShardId` differs from `shardId` only where a lineage +divided or folded an extent, and a hook that read `shardId` alone would search the source for a +shard the source does not hold; the convenience constructor defaults it to `shardId`. The target +epoch is the handoff's own rather than the plan's, so the two differ after a rebase. -`declare()` is read once per plan. +`declare()` is read once per plan. A `HookDeclaration` carries four components: the budget unit, and +whether the hooks support rollback, verification, and lineage. ```java HookDeclaration declaration = HookDeclaration.of("rows"); -HookDeclaration withoutVerification = new HookDeclaration("rows", true, false); +HookDeclaration dividing = HookDeclaration.withLineage("rows"); +HookDeclaration withoutVerification = new HookDeclaration("rows", true, false, false); ``` `budgetUnit` is opaque to the library: it is reported and never interpreted, and the unit counts a transfer and a catch-up answer are summed and compared and nothing else. `supportsVerify` of false is what lets `cleanup` follow a committed cutover directly, with no `verify` call. +`supportsLineage` states whether the integrator's storage can divide a copy in place and fold two +adjacent copies back together, which is a property of that storage rather than of the strategy. `of` +answers false for it and `withLineage` answers true. A plan that needs a local step over hooks that +declare false is refused with the cause `lineageUnsupported`, rather than reaching `DIVIDING` with +no hook to call and no way back. + `HookResult` is sealed over `Success`, `Deferred`, `Retryable`, and `Permanent`. `Retryable` is retried up to `maxAttemptsPerStep` with the policy's backoff, `Permanent` is never retried, and `Deferred` counts against no attempt budget and is not retried before its interval has elapsed on @@ -699,12 +976,13 @@ and `Permanent` are the two failures. ### The handoff states -`HandoffState` holds eleven values. `COMPLETE`, `ABORTED`, and `FAILED` are terminal, and nothing +`HandoffState` holds twelve values. `COMPLETE`, `ABORTED`, and `FAILED` are terminal, and nothing leaves a terminal state except a re-observation the integrator calls for one named handoff. | State | What is happening | |---|---| | `PLANNED` | admitted to the plan, no hook called | +| `DIVIDING` | a node holding both the parent and the child is dividing or folding its own copy | | `PREPARING` | the destination is being made ready to receive | | `TRANSFERRING` | the bulk contents are being copied | | `CATCHING_UP` | the residue accumulated during the copy is being closed | @@ -728,7 +1006,11 @@ that horizon. Mutual exclusion across a cutover therefore rests on the single-wi assumption about the integrator's two clocks that the library states and cannot verify. `cleanup` never runs before the copy is established, and `rollback` never runs after a cutover -record belonging to the handoff exists. +record belonging to the handoff exists. A local step aborted in `DIVIDING` is undone by the inverse +hook, a division by `combine` and a fold by `divide`. Where the inverse fails and its attempts are +spent, the handoff reaches `FAILED` with the failure kind `undivided`, which names a copy on one +node matching neither the parent's extent nor the child's; each of the other four kinds names a +condition of a copy that moved between nodes. ### Rate control diff --git a/ports/java/conformance/report.txt b/ports/java/conformance/report.txt index 39da880..6ace4a9 100644 --- a/ports/java/conformance/report.txt +++ b/ports/java/conformance/report.txt @@ -1,17 +1,17 @@ sharder conformance report, Java port -suite revision e91417161ae0de0002cf3748a9ecd969e82c6233f0d8ab97fd3e908492b24204 +suite revision c29e6a8f3facafa56414e46851ada1610db5e64c016714a8dba915e08875d57c - hash 2 of 2 vector files, 34 of 34 cases, 0.00s - place 45 of 45 vector files, 291 of 291 cases, 0.04s - core 8 of 8 vector files, 133 of 133 cases, 7.35s - scale 2 of 2 vector files, 2 of 2 cases, 0.83s + hash 2 of 2 vector files, 34 of 34 cases, 0.01s + place 45 of 45 vector files, 291 of 291 cases, 0.09s + core 8 of 8 vector files, 135 of 135 cases, 7.92s + scale 2 of 2 vector files, 2 of 2 cases, 0.87s failover 3 of 3 vector files, 67 of 67 cases, 0.00s - readAffinity 1 of 1 vector files, 46 of 46 cases, 0.00s + readAffinity 1 of 1 vector files, 46 of 46 cases, 0.01s fencing 3 of 3 vector files, 8 of 8 cases, 0.00s - migration 3 of 3 vector files, 15 of 15 cases, 0.00s + migration 5 of 5 vector files, 26 of 26 cases, 0.01s - 596 cases, 0 failures - scale wall time 0.83s - scale wall time 833 ms - peak resident size 683659264 bytes (651 MiB) + 609 cases, 0 failures + scale wall time 0.87s + scale wall time 869 ms + peak resident size 614211584 bytes (585 MiB) runtime OpenJDK 64-Bit Server VM 25.0.4 diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java index d33bc42..689c227 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/Sharder.java @@ -2,6 +2,7 @@ import com.codeheadsystems.sharder.Router; import com.codeheadsystems.sharder.config.RouterConfig; +import com.codeheadsystems.sharder.core.internal.observe.MetricsHolder; import com.codeheadsystems.sharder.core.internal.route.DefaultRouter; import com.codeheadsystems.sharder.migrate.HandoffCoordinator; import com.codeheadsystems.sharder.migrate.internal.DefaultCoordinator; @@ -34,6 +35,21 @@ public static HandoffCoordinator coordinator() { return new DefaultCoordinator(); } + /** + * The same, reporting through the registry and the sink a configuration carries. + * + *

A coordinator reads the two observability members of the configuration and nothing else + * from it: no provider, no snapshot, and no health view. Where either is unset it reports as + * the sinkless coordinator does, which is to count by name and discard. The events are the + * {@code migration.} rows of {@code OBS-020}, and + * {@code adr/0092} records why the sink arrives this way. + */ + public static HandoffCoordinator coordinator(RouterConfig config) { + return new DefaultCoordinator(new MetricsHolder( + config.observability().metricsRegistry(), config.observability().eventSink()), + config.clock()); + } + /** * A loader that validates a document without installing it, under {@code TOPO-191}. * diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java index 37dc5f2..8527a78 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/document/TopologyLoader.java @@ -55,6 +55,8 @@ public record Arrival(Outcome outcome, Optional condition, Optional strategies; private final Map retained = new LinkedHashMap<>(); + private final Optional configuredTopologyId; + private final OptionalLong minEpoch; private String topologyId; private TopologyDocument inForce; private Digest digestInForce; @@ -68,7 +70,23 @@ public TopologyLoader() { /** A pipeline under the strategies a configuration registered, under {@code CORE-010}. */ public TopologyLoader(Map strategies) { + this(strategies, Optional.empty(), OptionalLong.empty()); + } + + /** + * The same, under the identifier and the epoch floor the integrator configured. + * + *

{@code TOPO-061} checks a configured identifier before there is a snapshot in force, so a + * first document under a foreign identifier is a conflict rather than an installation, and + * {@code TOPO-071} applies {@code minEpoch} to every document including the first after a + * restart. Both are refused before the row that installs a first document, which is why they + * belong to the pipeline rather than to a caller checking afterwards. + */ + public TopologyLoader(Map strategies, + Optional configuredTopologyId, OptionalLong minEpoch) { this.strategies = Map.copyOf(strategies); + this.configuredTopologyId = configuredTopologyId; + this.minEpoch = minEpoch; } /** The snapshot in force, where one is. */ @@ -86,9 +104,16 @@ public OptionalLong epochInForce() { return epochInForce; } - /** The identifier every later document is checked against, adopted from the first accepted. */ + /** + * The identifier every document is checked against. + * + *

It is the one the integrator configured where there is one, and otherwise the one adopted + * from the first accepted document under {@code TOPO-091}. + */ public Optional expectedTopologyId() { - return Optional.ofNullable(topologyId); + return configuredTopologyId.isPresent() + ? configuredTopologyId + : Optional.ofNullable(topologyId); } /** The document's arrival, which installs it, treats it as a no-op, or refuses it. */ @@ -101,10 +126,18 @@ public Arrival accept(JsonObject document) { } TopologyDocument parsed = TopologyDocument.parse(document); - // TOPO-061, in the order the table writes the rows. - if (topologyId != null && !topologyId.equals(parsed.topologyId())) { + // TOPO-061, in the order the table writes the rows. The first two precede the row that + // installs a first document, so a first document under a foreign identifier is a conflict + // and a first document below minEpoch is stale, rather than an installation. + Optional expected = expectedTopologyId(); + if (expected.isPresent() && !expected.get().equals(parsed.topologyId())) { return refused(ErrorCode.TOPOLOGY_CONFLICT, "a differing topologyId", digest); } + if (minEpoch.isPresent() && parsed.epoch() < minEpoch.getAsLong()) { + // TOPO-071: the floor holds across a restart, so it is checked whether or not a + // snapshot is in force. + return refused(ErrorCode.STALE_DOCUMENT, "an epoch below minEpoch", digest); + } if (inForce == null) { return install(parsed, digest); } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java index 1144600..e60364a 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/Handoff.java @@ -14,8 +14,37 @@ */ public final class Handoff { + /** + * Whether contents cross the network, or the node divides or folds a copy it already holds. + * + *

A local step under {@code LIN-051} names one node as both its source and its destination + * and runs the shorter sequence through {@code dividing}, because there is no second party to + * prepare, quiesce, or cut over to. + */ + public enum Kind { + HANDOFF("handoff"), DIVIDE("divide"), COMBINE("combine"); + + private final String spelling; + + Kind(String spelling) { + this.spelling = spelling; + } + + /** The spelling a vector joins on. */ + public String spelling() { + return spelling; + } + + /** Whether this is a local step rather than a move between nodes. */ + public boolean local() { + return this != HANDOFF; + } + } + private final String id; private final String shard; + private final String sourceShard; + private final Kind kind; private final NodeId source; private final NodeId destination; private final long fromEpoch; @@ -27,8 +56,20 @@ public final class Handoff { Handoff(String id, String shard, NodeId source, NodeId destination, long fromEpoch, long toEpoch) { + this(id, shard, shard, source, destination, fromEpoch, toEpoch); + } + + Handoff(String id, String shard, String sourceShard, NodeId source, NodeId destination, + long fromEpoch, long toEpoch) { + this(id, shard, sourceShard, Kind.HANDOFF, source, destination, fromEpoch, toEpoch); + } + + Handoff(String id, String shard, String sourceShard, Kind kind, NodeId source, + NodeId destination, long fromEpoch, long toEpoch) { this.id = id; this.shard = shard; + this.sourceShard = sourceShard; + this.kind = kind; this.source = source; this.destination = destination; this.fromEpoch = fromEpoch; @@ -45,6 +86,23 @@ public String shard() { return shard; } + /** + * The shard whose contents this handoff moves, which {@code LIN-041} draws from the lineage. + * + *

It is the shard itself wherever the two snapshots enumerate the same shards. Where the + * later snapshot divided or folded an extent, the contents live under the parent's identifier + * at the source, so a hook that looked only at {@link #shard()} would search the source for a + * shard it does not hold. + */ + public String sourceShard() { + return sourceShard; + } + + /** Whether contents move between nodes, or a node divides a copy it holds. */ + public Kind kind() { + return kind; + } + /** The node that owns the shard before the cutover. */ public NodeId source() { return source; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java index 70b2a35..e0d0e7e 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/migrate/MigrationPlan.java @@ -1,6 +1,7 @@ package com.codeheadsystems.sharder.core.internal.migrate; import com.codeheadsystems.sharder.NodeId; +import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; import com.codeheadsystems.sharder.migrate.HandoffState; @@ -93,9 +94,27 @@ public static Handoff handoffOf(String id, String shard, NodeId source, NodeId d return new Handoff(id, shard, source, destination, fromEpoch, toEpoch); } + /** The same, carrying the parent and the kind a lineage gives it under {@code LIN-041}. */ + public static Handoff handoffOf(String id, String shard, String sourceShard, Handoff.Kind kind, + NodeId source, NodeId destination, long fromEpoch, + long toEpoch) { + return new Handoff(id, shard, sourceShard, kind, source, destination, fromEpoch, toEpoch); + } + /** - * The plan between two snapshots: one handoff per shard that gained a node, under the - * ownership delta of {@code TOPO-211}. + * The plan between two snapshots, derived from the lineage under {@code LIN-041}. + * + *

One handoff is admitted for each shard of the later snapshot, each shard its extent draws + * from, and each destination that does not already hold that parent's contents. Where the two + * snapshots enumerate the same shards the lineage is the identity of {@code LIN-007} and every + * shard is its own parent, which is the ownership delta's answer and the case every topology + * the suite carried before the lineage falls into. + * + *

Building the plan over the delta alone is what {@code LIN-044} forbids. A shard that + * absorbed a folded extent keeps its replica set, so the delta reports no entry for it, and a + * destination that holds one parent but not the other would receive nothing. A shard divided + * out of a parent has no entry in the earlier snapshot at all, so the source would fall back to + * the destination and the plan would tell a node to copy from itself. * *

Plan construction is a pure function of the two snapshots and the policy, which is what * lets a restarted coordinator rebuild the same plan under {@code MOVE-221}. @@ -103,27 +122,72 @@ public static Handoff handoffOf(String id, String shard, NodeId source, NodeId d public static MigrationPlan of(PlacementEngine from, PlacementEngine to, long quiesceLeaseMarginMillis) { MigrationPlan plan = new MigrationPlan(quiesceLeaseMarginMillis, to.document().epoch()); - for (OwnershipDelta.ShardChange change : OwnershipDelta.between(from, to)) { - // A shard that gained a node and lost one is a move from that source to that - // destination; the pairing is by position, which the delta reports in identity order. - for (int entry = 0; entry < change.gained().size(); entry++) { - NodeId destination = change.gained().get(entry); - NodeId source = entry < change.lost().size() - ? change.lost().get(entry) - : change.before().isEmpty() ? destination : change.before().get(0); - // A handoff is named after the shard it moves, so a rebuilt plan names the same - // handoffs, which MOVE-221 rests on. - String id = "h-" + change.shard(); - while (plan.handoffs.containsKey(id)) { - id = id + "-" + entry; + Map> parents = ShardExtents.parents(from, to); + Map classes = new LinkedHashMap<>(); + for (ShardExtents.Entry entry : ShardExtents.classify(from, to, OwnershipDelta::replicas)) { + classes.put(entry.shard(), entry.lineage()); + } + for (String shard : to.placement().shards()) { + List destinations = OwnershipDelta.replicas(to, shard); + List drawn = parents.getOrDefault(shard, List.of(shard)); + ShardExtents.Lineage lineage = classes.get(shard); + + // LIN-057: a division is sequenced before every handoff that draws from the divided + // parent, and a fold after every handoff that draws into the folded shard. + if (lineage == ShardExtents.Lineage.DIVIDED) { + String parent = drawn.get(0); + for (NodeId node : destinations) { + if (OwnershipDelta.replicas(from, parent).contains(node)) { + admit(plan, shard, parent, Handoff.Kind.DIVIDE, node, node, from, to); + } + } + } + for (String parent : drawn) { + List held = OwnershipDelta.replicas(from, parent); + if (held.isEmpty()) { + // LIN-043: a shard with no parent has no contents to move. + continue; + } + List needing = new ArrayList<>(destinations); + needing.removeAll(held); + List departing = new ArrayList<>(held); + departing.removeAll(destinations); + for (int entry = 0; entry < needing.size(); entry++) { + // LIN-045: a replica giving the shard up is drained in preference to one + // keeping it, so a plan rebuilt from the same snapshots names the same source. + NodeId source = entry < departing.size() ? departing.get(entry) : held.get(0); + admit(plan, shard, parent, Handoff.Kind.HANDOFF, source, needing.get(entry), + from, to); + } + } + if (lineage == ShardExtents.Lineage.MERGED) { + for (NodeId node : destinations) { + boolean holdsAParent = drawn.stream() + .anyMatch(parent -> OwnershipDelta.replicas(from, parent).contains(node)); + if (holdsAParent) { + admit(plan, shard, shard, Handoff.Kind.COMBINE, node, node, from, to); + } } - plan.handoffs.put(id, new Handoff(id, change.shard(), source, destination, - from.document().epoch(), to.document().epoch())); } } return plan; } + /** One entry of a plan, named so that a rebuilt plan names it identically under MOVE-221. */ + private static void admit(MigrationPlan plan, String shard, String sourceShard, + Handoff.Kind kind, NodeId source, NodeId destination, + PlacementEngine from, PlacementEngine to) { + String prefix = kind.local() ? "l-" : "h-"; + // The suffix counts the entries already admitted for this shard whatever their kind, so a + // local step and the handoff beside it never collide and the numbering is positional. + long already = plan.handoffs.values().stream() + .filter(existing -> existing.shard().equals(shard)) + .count(); + String id = already == 0 ? prefix + shard : prefix + shard + "-" + already; + plan.handoffs.put(id, new Handoff(id, shard, sourceShard, kind, source, destination, + from.document().epoch(), to.document().epoch())); + } + /** The handoffs of the plan, in the order it admitted them. */ public List handoffs() { return List.copyOf(handoffs.keySet()); @@ -247,7 +311,14 @@ public StepOutcome step(String id, String trigger, long at, String pressure) { } HandoffState to = switch (trigger) { case "admittedByRatePolicy" -> require(from, HandoffState.PLANNED, - HandoffState.PREPARING); + handoff(id).kind().local() ? HandoffState.DIVIDING : HandoffState.PREPARING); + // LIN-051: the same admission, named for the state it reaches, which a scenario + // drives by trigger rather than by reading the handoff's kind. + case "admittedByRatePolicyLocal" -> require(from, HandoffState.PLANNED, + HandoffState.DIVIDING); + // LIN-051: a local step moves nothing between nodes, so it runs the short sequence. + case "divideSuccess", "combineSuccess" -> require(from, HandoffState.DIVIDING, + HandoffState.COMPLETE); case "prepareSuccess" -> require(from, HandoffState.PREPARING, HandoffState.TRANSFERRING); case "noBulkRemaining" -> require(from, HandoffState.TRANSFERRING, @@ -267,7 +338,9 @@ public StepOutcome step(String id, String trigger, long at, String pressure) { ? HandoffState.ABORTED : HandoffState.ABORTING; // MOVE-021: attempts exhausted short of the cutover is an abort rather than a failure. case "attemptsExhausted" -> switch (from) { - case PREPARING, TRANSFERRING, CATCHING_UP -> HandoffState.ABORTING; + // MOVE-421: a local step is undone by the inverse hook, so it aborts like the + // three states that already do, under LIN-056. + case PREPARING, TRANSFERRING, CATCHING_UP, DIVIDING -> HandoffState.ABORTING; default -> null; }; case "quiesceExpired" -> require(from, HandoffState.CUTOVER, HandoffState.ABORTING); @@ -291,6 +364,8 @@ private StepOutcome failure(Handoff handoff, HandoffState from, String trigger) String kind = switch (trigger) { case "verifyMismatch" -> "unverified"; case "commitUndetermined" -> "undetermined"; + // MOVE-011: a copy that matches neither the parent's extent nor the child's. + case "dividePermanent" -> "undivided"; // MOVE-011: the kind names which copy the failure leaves at risk, so an exhausted // attempt budget means one thing in verifying, another in cleanup, and another while // compensation runs. diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java index 2a4120e..4304fbc 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/observe/Observability.java @@ -60,6 +60,10 @@ public static List eventsOf(String surface) { /** The events of the {@code migration} surface. */ public static final List MIGRATION_EVENTS = List.of( + // LIN-031: the count per class, which is where an operator reads whether an + // epoch was a refinement or a redistribution. Under 0091 nothing acts on it. + event("migration.lineage", "info", "unchanged", "moved", "divided", "merged", + "fresh", "split", "folded", "vacated"), event("migration.planned", "info", "handoffCount", "policy"), event("migration.state_changed", "info", "handoff", "shard", "from", "to", "trigger"), event("migration.cutover_committed", "info", "shard", "source", "destination", diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java new file mode 100644 index 0000000..5d64103 --- /dev/null +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/placement/ShardExtents.java @@ -0,0 +1,404 @@ +package com.codeheadsystems.sharder.core.internal.placement; + +import com.codeheadsystems.sharder.core.internal.document.TopologyDocument; +import com.codeheadsystems.sharder.core.internal.hash.U64; +import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; +import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Shard extents and the lineage over them, under {@code LIN-001} through {@code LIN-034}. + * + *

The extent of a shard is the set of routing keys {@code shardOf} maps to it, determined by the + * document alone under {@code LIN-001}. Under {@code ring} an extent is the half-open token + * interval of {@code RING-020}, which wraps at zero, so containment is interval arithmetic over + * unsigned 64-bit bounds and every comparison here goes through {@link U64}. A signed comparison + * would read an extent near the top of the space as empty, which is why this class sits in the + * package {@code verifyUnsignedComparisons} covers rather than beside the coordinator. + * + *

Under {@code slot} {@code TOPO-231} holds {@code slotCount} equal, and under {@code directory} + * {@code LIN-013} refuses a pair whose shard sets differ, so both are the identity lineage of + * {@code LIN-007} and neither reaches the interval arithmetic. + */ +public final class ShardExtents { + + /** The class of one shard under {@code LIN-021}. */ + public enum Lineage { + UNCHANGED, MOVED, DIVIDED, MERGED, FRESH, SPLIT, FOLDED, VACATED + } + + /** One shard's class and the shards its extent draws from. */ + public record Entry(String shard, Lineage lineage, List parents) { + } + + /** A lineage that cannot be computed, carrying the cause {@code ERR-050} reports. */ + public static final class Refused extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final String cause; + + Refused(String cause, String detail) { + super(cause + ": " + detail); + this.cause = cause; + } + + public String cause() { + return cause; + } + } + + /** + * One inclusive segment {@code [lo, hi]} of the hash space, with unsigned 64-bit bounds. + * + *

Bounds are inclusive rather than half-open because the space ends at an unsigned value a + * {@code long} cannot hold one past, and an interval that wraps past zero is two segments, so + * every segment here has {@code lo} at or below {@code hi} and no arithmetic reasons about a + * wrap. Nothing measures a width: a width of the whole space overflows, so containment is + * decided by subtraction instead. + */ + private record Segment(long lo, long hi) { + } + + private ShardExtents() { + } + + /** The keys {@code h} with {@code lowExclusive < h <= highInclusive}, as segments. */ + private static List segments(long lowExclusive, long highInclusive) { + // An equality of two 64-bit values compiles to LCMP, which the placement path forbids, so + // the comparison goes through U64 like every other one here. + if (U64.compare(lowExclusive, highInclusive) == 0) { + // One token owns the whole space, from zero to the greatest unsigned value. + return List.of(new Segment(0L, -1L)); + } + long lo = lowExclusive + 1; + if (U64.compare(lo, highInclusive) <= 0) { + return List.of(new Segment(lo, highInclusive)); + } + // The interval wraps past zero, so it is the head of the space and its tail. + return List.of(new Segment(0L, highInclusive), new Segment(lo, -1L)); + } + + /** The extent of each shard the ring names, under {@code LIN-011}. */ + private static Map> ringExtents(List shards) { + Map> out = new LinkedHashMap<>(); + int count = shards.size(); + for (int index = 0; index < count; index++) { + long token = U64.parseHex(shards.get(index)); + long predecessor = count == 1 + ? token + : U64.parseHex(shards.get((index - 1 + count) % count)); + out.put(shards.get(index), segments(predecessor, token)); + } + return out; + } + + private static Optional overlap(Segment left, Segment right) { + long lo = U64.max(left.lo(), right.lo()); + long hi = U64.min(left.hi(), right.hi()); + return U64.compare(lo, hi) <= 0 ? Optional.of(new Segment(lo, hi)) : Optional.empty(); + } + + private static boolean meets(List first, List second) { + for (Segment left : first) { + for (Segment right : second) { + if (overlap(left, right).isPresent()) { + return true; + } + } + } + return false; + } + + /** What remains of {@code inner} once every segment of {@code outer} is taken out of it. */ + private static List subtract(List inner, List outer) { + List remaining = new ArrayList<>(inner); + for (Segment cut : outer) { + List next = new ArrayList<>(); + for (Segment piece : remaining) { + Optional shared = overlap(piece, cut); + if (shared.isEmpty()) { + next.add(piece); + continue; + } + Segment taken = shared.get(); + if (U64.compare(piece.lo(), taken.lo()) < 0) { + next.add(new Segment(piece.lo(), taken.lo() - 1)); + } + if (U64.compare(taken.hi(), piece.hi()) < 0) { + next.add(new Segment(taken.hi() + 1, piece.hi())); + } + } + remaining = next; + } + return remaining; + } + + private static boolean contains(List outer, List inner) { + return subtract(inner, outer).isEmpty(); + } + + private static boolean equal(List first, List second) { + return contains(first, second) && contains(second, first); + } + + /** + * One region of the keyspace the two directory tables cut it into, under {@code LIN-016}. + * + *

{@code exact} is the key equal to {@code node}; otherwise it is the keys strictly + * extending {@code node} that no deeper node of the combined trie is a prefix of. + */ + private record Region(boolean exact, String node) { + } + + /** Every decoded matcher value of either table, rendered as hexadecimal, and the empty one. */ + private static List regions(TopologyDocument before, TopologyDocument after) { + Set nodes = new java.util.TreeSet<>(); + nodes.add(""); + for (TopologyDocument document : List.of(before, after)) { + for (TopologyDocument.DirectoryEntry entry : document.strategy().entries()) { + nodes.add(HexFormat.of().formatHex(entry.match().value())); + } + } + List out = new ArrayList<>(); + nodes.stream().filter(node -> !node.isEmpty()) + .forEach(node -> out.add(new Region(true, node))); + nodes.forEach(node -> out.add(new Region(false, node))); + return List.copyOf(out); + } + + /** + * {@code PLACE-065}: the entry index winning a region, or {@code -1} where none matches it. + * + *

An {@code exact} matcher wins only the region that is its own key, because a region of + * keys strictly extending a node contains no node and every matcher value is one. + */ + private static int winner(TopologyDocument document, Region region) { + List entries = document.strategy().entries(); + HexFormat hex = HexFormat.of(); + if (region.exact()) { + for (int index = 0; index < entries.size(); index++) { + TopologyDocument.Matcher matcher = entries.get(index).match(); + if ("exact".equals(matcher.kind()) + && hex.formatHex(matcher.value()).equals(region.node())) { + return index; + } + } + } + int best = -1; + int bestLength = -1; + for (int index = 0; index < entries.size(); index++) { + TopologyDocument.Matcher matcher = entries.get(index).match(); + if (!"prefix".equals(matcher.kind())) { + continue; + } + String value = hex.formatHex(matcher.value()); + if (!region.node().startsWith(value)) { + continue; + } + if (value.length() > bestLength) { + best = index; + bestLength = value.length(); + } + } + return best; + } + + /** {@code LIN-013}: each shard's extent as the set of regions its entry wins. */ + private static Map> directoryExtents(PlacementEngine engine, + List regions) { + List shards = engine.placement().shards(); + Map> out = new LinkedHashMap<>(); + shards.forEach(shard -> out.put(shard, new LinkedHashSet<>())); + for (Region region : regions) { + int index = winner(engine.document(), region); + if (index >= 0) { + out.get(shards.get(index)).add(region); + } + } + return out; + } + + /** + * A shard's extent, whichever geometry decides it. + * + *

Under {@code ring} it is a set of intervals of the hash space; under {@code directory} it + * is a set of regions of the prefix trie. The three predicates below are what the + * classification reads, so nothing above this point depends on which kind is in play. + */ + private sealed interface Extent permits RingExtent, DirExtent { + } + + private record RingExtent(List segments) implements Extent { + } + + private record DirExtent(Set regions) implements Extent { + } + + private static boolean meetsExtent(Extent first, Extent second) { + if (first instanceof RingExtent left && second instanceof RingExtent right) { + return meets(left.segments(), right.segments()); + } + Set left = ((DirExtent) first).regions(); + Set right = ((DirExtent) second).regions(); + return left.stream().anyMatch(right::contains); + } + + private static boolean containsExtent(Extent outer, Extent inner) { + if (outer instanceof RingExtent left && inner instanceof RingExtent right) { + return contains(left.segments(), right.segments()); + } + return ((DirExtent) outer).regions().containsAll(((DirExtent) inner).regions()); + } + + private static boolean equalExtent(Extent first, Extent second) { + if (first instanceof RingExtent left && second instanceof RingExtent right) { + return equal(left.segments(), right.segments()); + } + return ((DirExtent) first).regions().equals(((DirExtent) second).regions()); + } + + /** The two extent maps, decided by the geometry the strategy kind gives. */ + private static List> geometry(PlacementEngine before, + PlacementEngine after) { + Map earlier = new LinkedHashMap<>(); + Map later = new LinkedHashMap<>(); + if ("directory".equals(after.document().strategy().kind())) { + List regions = regions(before.document(), after.document()); + directoryExtents(before, regions).forEach((k, v) -> earlier.put(k, new DirExtent(v))); + directoryExtents(after, regions).forEach((k, v) -> later.put(k, new DirExtent(v))); + } else { + ringExtents(before.placement().shards()) + .forEach((k, v) -> earlier.put(k, new RingExtent(v))); + ringExtents(after.placement().shards()) + .forEach((k, v) -> later.put(k, new RingExtent(v))); + } + return List.of(earlier, later); + } + + /** + * For each shard the later snapshot names, the shards of the earlier one its extent draws from, + * under {@code LIN-004}. + */ + public static Map> parents(PlacementEngine before, PlacementEngine after) { + List earlier = before.placement().shards(); + List later = after.placement().shards(); + if (earlier.equals(later) || "slot".equals(after.document().strategy().kind())) { + Map> identity = new LinkedHashMap<>(); + later.forEach(shard -> identity.put(shard, List.of(shard))); + return identity; + } + List> extents = geometry(before, after); + Map> out = new LinkedHashMap<>(); + for (String shard : later) { + List drawn = new ArrayList<>(); + for (String candidate : earlier) { + if (meetsExtent(extents.get(0).get(candidate), extents.get(1).get(shard))) { + drawn.add(candidate); + } + } + out.put(shard, List.copyOf(drawn)); + } + return out; + } + + /** + * The lineage of two snapshots, in the order {@code LIN-033} fixes. + * + *

{@code replicaSet} answers a shard's replica set under one snapshot, which separates + * {@code unchanged} from {@code moved} and is the only thing here that reads placement. + */ + public static List classify(PlacementEngine before, PlacementEngine after, + ReplicaSet replicaSet) { + Optional differing = OwnershipDelta.incomparable(before.document(), + after.document()); + if (differing.isPresent()) { + throw new Refused("incomparableShards", differing.get()); + } + String kind = after.document().strategy().kind(); + if ("rendezvous".equals(kind)) { + throw new Refused("strategyUnsupported", "rendezvous enumerates no shard"); + } + List earlier = before.placement().shards(); + List later = after.placement().shards(); + if ("slot".equals(kind) || earlier.equals(later)) { + // LIN-007 and LIN-012: an equal shard set is an equal extent set, and TOPO-231 has + // already refused a slot pair whose slotCount differs. + return identity(later, before, after, replicaSet); + } + + List> extents = geometry(before, after); + Map earlierExtents = extents.get(0); + Map laterExtents = extents.get(1); + Set enumeratedLater = new LinkedHashSet<>(later); + List entries = new ArrayList<>(); + for (String shard : later) { + Extent mine = laterExtents.get(shard); + List drawn = new ArrayList<>(); + for (String candidate : earlier) { + Extent theirs = earlierExtents.get(candidate); + if (!meetsExtent(theirs, mine)) { + continue; + } + if (!containsExtent(theirs, mine) && !containsExtent(mine, theirs)) { + throw new Refused("unalignedLineage", shard); + } + drawn.add(candidate); + } + if (drawn.isEmpty()) { + entries.add(new Entry(shard, Lineage.FRESH, List.of())); + } else if (drawn.size() > 1) { + entries.add(new Entry(shard, Lineage.MERGED, List.copyOf(drawn))); + } else if (equalExtent(earlierExtents.get(drawn.get(0)), mine)) { + entries.add(new Entry(shard, moved(shard, before, after, replicaSet), + List.copyOf(drawn))); + } else { + entries.add(new Entry(shard, Lineage.DIVIDED, List.copyOf(drawn))); + } + } + for (String shard : earlier) { + if (enumeratedLater.contains(shard)) { + continue; + } + Extent mine = earlierExtents.get(shard); + List children = new ArrayList<>(); + for (String candidate : later) { + if (meetsExtent(laterExtents.get(candidate), mine)) { + children.add(candidate); + } + } + Lineage lineage = children.isEmpty() ? Lineage.VACATED + : children.size() > 1 ? Lineage.SPLIT : Lineage.FOLDED; + entries.add(new Entry(shard, lineage, List.copyOf(children))); + } + return List.copyOf(entries); + } + + private static List identity(List shards, PlacementEngine before, + PlacementEngine after, ReplicaSet replicaSet) { + List entries = new ArrayList<>(); + for (String shard : shards) { + entries.add(new Entry(shard, moved(shard, before, after, replicaSet), List.of(shard))); + } + return List.copyOf(entries); + } + + private static Lineage moved(String shard, PlacementEngine before, PlacementEngine after, + ReplicaSet replicaSet) { + return replicaSet.of(before, shard).equals(replicaSet.of(after, shard)) + ? Lineage.UNCHANGED : Lineage.MOVED; + } + + /** A shard's replica set under one snapshot, which {@code TOPO-211} defines. */ + @FunctionalInterface + public interface ReplicaSet { + List of(PlacementEngine engine, String shard); + } +} diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java index 803aad0..52ffccd 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/DefaultRouter.java @@ -101,7 +101,12 @@ public final class DefaultRouter implements Router { public DefaultRouter(RouterConfig config) { this.config = config; this.provider = config.provider().provider(); - this.pipeline = new TopologyLoader(config.strategies()); + // TOPO-061 and TOPO-071: the configured identifier and the epoch floor belong to the + // pipeline that installs, because both are refused before the row that installs a first + // document. The standalone loader of `loader()` checks one document in isolation and + // applies neither. + this.pipeline = new TopologyLoader(config.strategies(), + config.provider().expectedTopologyId(), config.provider().minEpoch()); this.health = config.healthView().orElseGet( () -> new SlidingWindowHealthView(config.health(), config.hintObserver().orElse(null))); diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java index b028df8..35073ac 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/core/internal/route/OwnershipDelta.java @@ -51,14 +51,27 @@ public static Optional incomparable(TopologyDocument before, TopologyDoc } /** - * The shards whose replica sets differ, in the enumeration order of the later snapshot. + * The shards whose replica sets differ, in the order {@code TOPO-213} fixes: first the shards + * the later snapshot enumerates, in its own enumeration order, then the shards only the + * earlier snapshot enumerates, in that snapshot's enumeration order. * - *

A shard the later snapshot does not enumerate is absent from the delta, and a shard the - * earlier one did not enumerate has an empty before set. + *

A shard the earlier snapshot did not enumerate has an empty before set, and a shard the + * later one does not enumerate has an empty after set. Neither is empty on both sides, so a + * shard that vanished reports every node it lost. Where the two snapshots enumerate the same + * shards, which is every pair under {@code slot} at one {@code slotCount}, the second group is + * empty and the order is the later snapshot's alone. */ public static List between(PlacementEngine before, PlacementEngine after) { List changes = new ArrayList<>(); - for (String shard : after.placement().shards()) { + List later = after.placement().shards(); + Set enumeratedLater = new LinkedHashSet<>(later); + List ordered = new ArrayList<>(later); + for (String shard : before.placement().shards()) { + if (!enumeratedLater.contains(shard)) { + ordered.add(shard); + } + } + for (String shard : ordered) { List was = replicas(before, shard); List now = replicas(after, shard); if (was.equals(now)) { diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java b/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java index ccc6d48..93f1656 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/error/ErrorCode.java @@ -168,9 +168,9 @@ public List causes() { "retryBudget"); case PLAN_REFUSED -> List.of("incomparableShards", "epochNotAdvancing", "strategyUnsupported", "destinationOutsidePlacementSet", "policyInvalid", - "topologyMismatch"); + "topologyMismatch", "unalignedLineage", "lineageUnsupported"); case HANDOFF_FAILED -> List.of("unverified", "residue", "undetermined", - "rollbackFailed"); + "rollbackFailed", "undivided"); default -> List.of(); }; } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java b/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java index 5193d47..cd4d60f 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/error/PlanRefusedException.java @@ -23,7 +23,11 @@ public enum Cause { /** The policy carries a value outside its range. */ POLICY_INVALID("policyInvalid"), /** The plan names a topology the router does not hold. */ - TOPOLOGY_MISMATCH("topologyMismatch"); + TOPOLOGY_MISMATCH("topologyMismatch"), + /** The change moves a boundary without dividing or folding an extent, {@code LIN-022}. */ + UNALIGNED_LINEAGE("unalignedLineage"), + /** A local step is needed and the storage declares none, under {@code LIN-053}. */ + LINEAGE_UNSUPPORTED("lineageUnsupported"); private final String spelling; diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java index 4c4141a..89d2e3c 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffContext.java @@ -10,6 +10,20 @@ * entered {@code cutover} keeps the epoch it held at that transition and runs to a terminal state * under it, under {@code MOVE-099}, so the two differ after a rebase. */ -public record HandoffContext(ShardId shardId, String topologyId, long fromEpoch, long toEpoch, - NodeId source, NodeId destination, int attempt, int deadlineMillis) { +public record HandoffContext(ShardId shardId, ShardId sourceShardId, String topologyId, + long fromEpoch, long toEpoch, NodeId source, NodeId destination, + int attempt, int deadlineMillis) { + + /** + * The context of a handoff whose contents come from the shard itself. + * + *

{@code sourceShardId} differs from {@code shardId} only where a lineage divided or folded + * an extent, under {@code LIN-041}. A hook that read {@code shardId} alone would search the + * source for a shard the source does not hold. + */ + public HandoffContext(ShardId shardId, String topologyId, long fromEpoch, long toEpoch, + NodeId source, NodeId destination, int attempt, int deadlineMillis) { + this(shardId, shardId, topologyId, fromEpoch, toEpoch, source, destination, attempt, + deadlineMillis); + } } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java index 3dfb8ef..b4292d9 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffCoordinator.java @@ -9,7 +9,6 @@ * it is given, computes the ownership delta between them, and answers a plan the integrator drives. * A plan is never created as a side effect of installing a snapshot, under {@code MOVE-101}. */ -@FunctionalInterface public interface HandoffCoordinator { /** @@ -22,4 +21,20 @@ public interface HandoffCoordinator { */ MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHooks hooks, MigrationPolicy policy); + + /** + * Where each shard's contents come from across the two snapshots, under {@code LIN-031}. + * + *

An integrator sizes a migration before deciding to run it, which is the reason + * {@code TOPO-212} makes the ownership delta a call rather than a product of installation, and + * the reason this is one too. Computing a lineage installs nothing and plans nothing. + * + *

It refuses with {@code PlanRefusedException} where the two snapshots carry differing + * {@code topologyId}s or do not join on shard identity, both under the cause + * {@code incomparableShards} of {@code TOPO-231}; where the strategy enumerates no shard, + * under {@code strategyUnsupported}; and where a boundary moved without either dividing or + * folding an extent whole, under {@code unalignedLineage} of {@code LIN-022}. A snapshot this + * library did not produce is an {@code InvalidArgumentException}, as it is for {@code plan}. + */ + ShardLineage lineage(TopologySnapshot from, TopologySnapshot to); } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java index 014a5b2..22d3499 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HandoffState.java @@ -1,7 +1,7 @@ package com.codeheadsystems.sharder.migrate; /** - * The eleven handoff states of {@code MOVE-001}. + * The twelve handoff states of {@code MOVE-001}. * *

{@code complete}, {@code aborted}, and {@code failed} are terminal, and nothing transitions * out of one except the re-observation of {@code MOVE-233}, which the integrator calls for one @@ -10,6 +10,8 @@ public enum HandoffState { /** Admitted to the plan, no hook called. */ PLANNED("planned"), + /** A node holding both the parent and the child is dividing or folding its own copy. */ + DIVIDING("dividing"), /** The destination is being made ready to receive. */ PREPARING("preparing"), /** The bulk contents are being copied. */ diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java index 4ab3ea7..a89282a 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/HookDeclaration.java @@ -6,12 +6,23 @@ *

{@code budgetUnit} is opaque to the library: it is reported and never interpreted, under * {@code MOVE-141}. {@code supportsVerify} of false is what {@code MOVE-181} reads when it decides * whether {@code cleanup} may follow a committed cutover directly. + * + *

{@code supportsLineage} states whether the integrator's storage can divide a copy in place and + * fold two adjacent copies back together. It is a property of that storage rather than of this + * library, which is why it is declared here rather than derived from the strategy: {@code LIN-053} + * refuses a plan that needs a local step where it is false, in preference to calling a hook that + * was never implemented. */ public record HookDeclaration(String budgetUnit, boolean supportsRollback, - boolean supportsVerify) { + boolean supportsVerify, boolean supportsLineage) { - /** A declaration in units of {@code budgetUnit} supporting both rollback and verification. */ + /** A declaration in units of {@code budgetUnit} supporting rollback and verification. */ public static HookDeclaration of(String budgetUnit) { - return new HookDeclaration(budgetUnit, true, true); + return new HookDeclaration(budgetUnit, true, true, false); + } + + /** The same, and a storage that can divide a copy in place under {@code LIN-053}. */ + public static HookDeclaration withLineage(String budgetUnit) { + return new HookDeclaration(budgetUnit, true, true, true); } } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java new file mode 100644 index 0000000..4f073c7 --- /dev/null +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/LineageClass.java @@ -0,0 +1,44 @@ +package com.codeheadsystems.sharder.migrate; + +/** + * What became of one shard between two snapshots, under {@code LIN-021}. + * + *

A shard the later snapshot enumerates is classified against the earlier one, and a shard only + * the earlier snapshot enumerates is classified against the later one, so every shard of a lineage + * carries exactly one of these. A pair whose extents neither divide nor fold whole has no class at + * all: it is refused under {@code LIN-022} rather than classified. + */ +public enum LineageClass { + /** One parent, whose extent is equal, and the replica set is equal. */ + UNCHANGED("unchanged"), + /** One parent, whose extent is equal, and the replica set differs. */ + MOVED("moved"), + /** One parent, whose extent strictly contains this one. */ + DIVIDED("divided"), + /** Two or more parents, whose extents this one contains. */ + MERGED("merged"), + /** No parent: this extent meets no extent of the earlier snapshot. */ + FRESH("fresh"), + /** Only the earlier snapshot enumerates it, and two or more children divide its extent. */ + SPLIT("split"), + /** Only the earlier snapshot enumerates it, and one child's extent contains its own. */ + FOLDED("folded"), + /** Only the earlier snapshot enumerates it, and its extent meets no later extent. */ + VACATED("vacated"); + + private final String spelling; + + LineageClass(String spelling) { + this.spelling = spelling; + } + + /** The spelling a vector, a scenario, and an event join on. */ + public String spelling() { + return spelling; + } + + /** Whether the shard's extent is the one its parent held. */ + public boolean extentUnchanged() { + return this == UNCHANGED || this == MOVED; + } +} diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java index 14bd055..56e9a67 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/MovementHooks.java @@ -41,6 +41,28 @@ public interface MovementHooks { /** Compensates an abort, under {@code MOVE-191}. */ HookResult rollback(HandoffContext context); + /** + * Divides the copy this node holds so that it matches the extent of {@code shardId}. + * + *

A local step under {@code LIN-051}: the node holds {@code sourceShardId} under the earlier + * snapshot and {@code shardId} under the later one, and nothing moves between nodes. The + * default refuses, because a storage that has not implemented it declares + * {@code supportsLineage} of false and {@code LIN-053} refuses the plan before a call is made. + */ + default HookResult divide(HandoffContext context) { + return new HookResult.Permanent("divide is not implemented"); + } + + /** + * Folds the copies this node holds into one that matches the extent of {@code shardId}. + * + *

The inverse of {@link #divide}, and what {@code LIN-056} calls to undo a division that was + * aborted, which is why a storage declares support for the pair rather than for either alone. + */ + default HookResult combine(HandoffContext context) { + return new HookResult.Permanent("combine is not implemented"); + } + /** Reads the durable state, under {@code MOVE-201}. */ ObserveResult observe(HandoffContext context); } diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java new file mode 100644 index 0000000..482b784 --- /dev/null +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/ShardLineage.java @@ -0,0 +1,70 @@ +package com.codeheadsystems.sharder.migrate; + +import com.codeheadsystems.sharder.ShardId; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +/** + * Where each shard's contents come from across an epoch, under {@code LIN-031}. + * + *

An ownership delta joins two snapshots on the shard identifier and answers which shards + * changed owner. Where an epoch changes which shards exist, that join has no answer for the ones + * that appeared or vanished. A lineage is the second join, over the keys a shard holds rather than + * over its name, and it is what a plan reads to decide where a divided shard's contents are. + * + *

It is derived from the two topology documents. No member of a document records it, and an + * implementation never computes one by sampling routing keys, under {@code LIN-001}. + * + *

Where the two snapshots enumerate the same shards the lineage is the identity: every shard is + * its own parent and nothing is divided, merged, fresh, or vacated. That is every pair under + * {@code slot}, and every pair under any kind whose shard set did not move. + */ +public record ShardLineage(List entries) { + + /** + * One shard's class and the shards its extent draws from. + * + *

{@code parents} names the shards of the earlier snapshot for an entry the later snapshot + * enumerates, and the shards of the later snapshot for one only the earlier snapshot + * enumerates. It is empty for a shard that is {@code FRESH} or {@code VACATED}. + */ + public record Entry(ShardId shard, LineageClass lineage, List parents) { + + /** The lists are copied, so a caller holds an entry across an installation unchanged. */ + public Entry { + parents = List.copyOf(parents); + } + } + + /** The entries are copied, in the order {@code LIN-033} fixes. */ + public ShardLineage { + entries = List.copyOf(entries); + } + + /** The entry for one shard, where the lineage carries one. */ + public Optional entry(ShardId shard) { + return entries.stream().filter(entry -> entry.shard().equals(shard)).findFirst(); + } + + /** + * How many shards fall in each class, which is what {@code migration.lineage} reports. + * + *

An operator reads this to see whether an epoch refined the keyspace or redistributed it. + * The library takes no position on which it should have been, under + * {@code adr/0091}. + */ + public Map counts() { + Map out = new TreeMap<>(); + for (Entry entry : entries) { + out.merge(entry.lineage(), 1L, Long::sum); + } + return Map.copyOf(out); + } + + /** Whether every shard kept the extent it held, which is the identity of {@code LIN-007}. */ + public boolean identity() { + return entries.stream().allMatch(entry -> entry.lineage().extentUnchanged()); + } +} diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java index 95b34d5..8b1ca9e 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultCoordinator.java @@ -1,15 +1,27 @@ package com.codeheadsystems.sharder.migrate.internal; import com.codeheadsystems.sharder.NodeId; +import com.codeheadsystems.sharder.ShardId; +import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; +import com.codeheadsystems.sharder.MonotonicClock; +import com.codeheadsystems.sharder.core.internal.observe.MetricsHolder; import com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot; import com.codeheadsystems.sharder.error.InvalidArgumentException; import com.codeheadsystems.sharder.error.PlanRefusedException; import com.codeheadsystems.sharder.migrate.HandoffCoordinator; import com.codeheadsystems.sharder.migrate.MigrationPlan; import com.codeheadsystems.sharder.migrate.MigrationPolicy; +import com.codeheadsystems.sharder.migrate.LineageClass; +import com.codeheadsystems.sharder.observe.Event; +import com.codeheadsystems.sharder.observe.Severity; import com.codeheadsystems.sharder.migrate.MovementHooks; +import com.codeheadsystems.sharder.migrate.ShardLineage; import com.codeheadsystems.sharder.topology.TopologySnapshot; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; /** * Where a plan is built, under {@code MOVE-061}. @@ -21,8 +33,23 @@ */ public final class DefaultCoordinator implements HandoffCoordinator { - /** The coordinator every plan is built through. */ + private final MetricsHolder metrics; + private final MonotonicClock clock; + + /** A coordinator that reports nothing, which is what {@code Sharder.coordinator()} answers. */ public DefaultCoordinator() { + this(new MetricsHolder(Optional.empty(), Optional.empty()), MonotonicClock.systemNanoTime()); + } + + /** + * A coordinator reporting through one sink, under {@code adr/0092}. + * + *

The clock stamps the events of {@code OBS-021} and nothing else: a plan reads no clock, + * and {@code step} takes the one the integrator drives it with. + */ + public DefaultCoordinator(MetricsHolder metrics, MonotonicClock clock) { + this.metrics = metrics; + this.clock = clock; } @Override @@ -41,8 +68,10 @@ public MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHo if (!target.engine().supportsOrchestratedMigration()) { throw new PlanRefusedException(PlanRefusedException.Cause.STRATEGY_UNSUPPORTED); } - var machine = com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.of( - source.engine(), target.engine(), policy.quiesceLeaseMarginMillis()); + // LIN-022: a boundary that neither divides nor folds an extent whole has no lineage to + // name, and LIN-013 refuses a directory pair whose shard sets differ until its extents are + // defined. Both surface here, before any handoff is admitted. + var machine = planOver(source, target, policy); machine.policy(policy.maxConcurrentHandoffs(), policy.maxConcurrentPerSourceNode(), policy.maxConcurrentPerDestinationNode()); // MOVE-333 and MOVE-336 both read the deadline, so the policy's value reaches the machine @@ -55,7 +84,77 @@ public MigrationPlan plan(TopologySnapshot from, TopologySnapshot to, MovementHo PlanRefusedException.Cause.DESTINATION_OUTSIDE_PLACEMENT_SET); } } - return new DefaultMigrationPlan(machine, source, target, hooks, policy); + // LIN-053: whether the storage can divide a copy in place is the integrator's statement, + // so a plan needing a local step is refused rather than calling a hook that is not there. + boolean needsLocalStep = machine.handoffs().stream() + .anyMatch(id -> machine.handoff(id).kind().local()); + if (needsLocalStep && !hooks.declare().supportsLineage()) { + throw new PlanRefusedException(PlanRefusedException.Cause.LINEAGE_UNSUPPORTED); + } + metrics.emit(new Event("sharder.migration.planned", clock.millis(), source.topologyId(), + target.epoch(), Severity.INFO, + Map.of("handoffCount", Integer.toString(machine.handoffs().size()), + "policy", policy.toString()))); + return new DefaultMigrationPlan(machine, source, target, hooks, policy, metrics, clock); + } + + /** The plan, with a lineage refusal reported as the condition {@code ERR-050} names. */ + private static com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan planOver( + DocumentSnapshot source, DocumentSnapshot target, MigrationPolicy policy) { + try { + return com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.of( + source.engine(), target.engine(), policy.quiesceLeaseMarginMillis()); + } catch (ShardExtents.Refused refusal) { + throw new PlanRefusedException(causeOf(refusal)); + } + } + + @Override + public ShardLineage lineage(TopologySnapshot from, TopologySnapshot to) { + DocumentSnapshot source = snapshot(from, "from"); + DocumentSnapshot target = snapshot(to, "to"); + // The same refusal `plan` gives for the same input. `TOPO-211` and `LIN-004` both join two + // snapshots of one `topologyId`, and a pair under two identifiers joins on nothing, which + // is the condition `TOPO-231` names rather than the plan-against-router mismatch. + if (!source.topologyId().equals(target.topologyId())) { + throw new PlanRefusedException(PlanRefusedException.Cause.INCOMPARABLE_SHARDS); + } + List entries = new ArrayList<>(); + for (ShardExtents.Entry entry : classify(source, target)) { + List parents = entry.parents().stream().map(ShardId::of).toList(); + entries.add(new ShardLineage.Entry(ShardId.of(entry.shard()), + LineageClass.valueOf(entry.lineage().name()), parents)); + } + ShardLineage lineage = new ShardLineage(entries); + Map payload = new java.util.LinkedHashMap<>(); + for (LineageClass value : LineageClass.values()) { + payload.put(value.spelling(), + Long.toString(lineage.counts().getOrDefault(value, 0L))); + } + metrics.emit(new Event("sharder.migration.lineage", clock.millis(), source.topologyId(), + target.epoch(), Severity.INFO, payload)); + return lineage; + } + + /** The classification, with a lineage refusal reported as the condition {@code ERR-050} names. */ + private static List classify(DocumentSnapshot source, + DocumentSnapshot target) { + try { + return ShardExtents.classify(source.engine(), target.engine(), + OwnershipDelta::replicas); + } catch (ShardExtents.Refused refusal) { + throw new PlanRefusedException(causeOf(refusal)); + } + } + + /** The closed-set cause of {@code ERR-050} that a lineage refusal carries. */ + private static PlanRefusedException.Cause causeOf(ShardExtents.Refused refusal) { + return switch (refusal.cause()) { + case "unalignedLineage" -> PlanRefusedException.Cause.UNALIGNED_LINEAGE; + case "lineageUnsupported" -> PlanRefusedException.Cause.LINEAGE_UNSUPPORTED; + case "strategyUnsupported" -> PlanRefusedException.Cause.STRATEGY_UNSUPPORTED; + default -> PlanRefusedException.Cause.INCOMPARABLE_SHARDS; + }; } private static DocumentSnapshot snapshot(TopologySnapshot snapshot, String name) { diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java index 4a2e3af..5b5999b 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/migrate/internal/DefaultMigrationPlan.java @@ -5,6 +5,10 @@ import com.codeheadsystems.sharder.core.internal.migrate.Handoff; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; +import com.codeheadsystems.sharder.MonotonicClock; +import com.codeheadsystems.sharder.core.internal.observe.MetricsHolder; +import com.codeheadsystems.sharder.observe.Event; +import com.codeheadsystems.sharder.observe.Severity; import com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot; import com.codeheadsystems.sharder.error.InvalidArgumentException; import com.codeheadsystems.sharder.error.PlanRefusedException; @@ -94,11 +98,17 @@ private static final class Driving { private volatile DocumentSnapshot target; private volatile com.codeheadsystems.sharder.topology.OwnershipDelta delta; + private final MetricsHolder metrics; + // OBS-021 stamps every event with an instant. It is not the clock `step` is driven with: a + // caller advances a plan on its own clock and an event records when it was reported. + private final MonotonicClock eventClock; /** The plan over one machine, its hooks, and the policy that bounds it. */ DefaultMigrationPlan(com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan machine, DocumentSnapshot source, DocumentSnapshot target, MovementHooks hooks, - MigrationPolicy policy) { + MigrationPolicy policy, MetricsHolder metrics, MonotonicClock eventClock) { + this.metrics = metrics; + this.eventClock = eventClock; this.machine = machine; this.source = source; this.target = target; @@ -110,6 +120,12 @@ private static final class Driving { machine.handoffs().forEach(id -> driving.put(id, new Driving())); } + /** One {@code migration.} event of {@code OBS-020}, stamped as {@code OBS-021} requires. */ + private void report(String name, Severity severity, Map payload) { + metrics.emit(new Event("sharder.migration." + name, eventClock.millis(), + source.topologyId(), target.epoch(), severity, payload)); + } + @Override public com.codeheadsystems.sharder.topology.OwnershipDelta delta() { return delta; @@ -170,6 +186,34 @@ public StepOutcome step(MonotonicClock clock) { return new StepOutcome.Idle(); } + /** + * {@code OBS-020}: the transition, and the failure where one is reached. + * + *

{@code migration.state_changed} fires on every transition the machine took, and a + * transition it dropped because the state had moved under a hook is not one, so nothing is + * reported for it. A handoff reaching {@code failed} carries its kind, which is what an + * operator acts on. + */ + private void reportTransition(String id, + String trigger, + com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.StepOutcome outcome) { + if (!"advanced".equals(outcome.outcome()) && !"settled".equals(outcome.outcome())) { + return; + } + report("state_changed", Severity.INFO, Map.of("handoff", id, + "shard", machine.handoff(id).shard(), + "from", outcome.fromState(), + "to", outcome.toState(), + "trigger", trigger)); + if (HandoffState.FAILED.spelling().equals(outcome.toState())) { + Handoff handoff = machine.handoff(id); + report("failed", Severity.ERROR, Map.of("shard", handoff.shard(), + "kind", handoff.failureKind().orElse("unknown"), + "source", handoff.source().asText(), + "destination", handoff.destination().asText())); + } + } + /** * One transition of the machine, taken under its lock. * @@ -188,7 +232,9 @@ private com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.StepOutc synchronized (machineLock) { HandoffState from = machine.handoff(id).state(); try { - return machine.step(id, trigger, at, pressure); + var outcome = machine.step(id, trigger, at, pressure); + reportTransition(id, trigger, outcome); + return outcome; } catch (IllegalStateException moved) { // The state named no transition for this trigger, which is what a concurrent // abort or recovery leaves behind. @@ -230,6 +276,7 @@ private StepOutcome advance(String id, Handoff handoff, Driving state, long at, HandoffState from = handoff.state(); return switch (from) { case PLANNED -> admit(id, handoff, level, at); + case DIVIDING -> divide(id, handoff, state, at); case PREPARING -> afterHook(id, state, at, from, hooks.prepare(context(handoff, state)), "prepareSuccess", "attemptsExhausted"); @@ -261,7 +308,25 @@ private StepOutcome admit(String id, Handoff handoff, PressureLevel level, long return null; } return new StepOutcome.Advanced(HandoffId.of(id), HandoffState.PLANNED, - HandoffState.PREPARING); + handoff.kind().local() ? HandoffState.DIVIDING : HandoffState.PREPARING); + } + + /** + * {@code LIN-052}: the local step, which divides a copy or folds several into one. + * + *

Nothing moves between nodes, so there is no destination to prepare and no cutover to + * commit. A permanent answer is the one case that reaches {@code failed} directly, with the + * kind {@code undivided} of {@code MOVE-011}, because the copy matches neither extent. + */ + private StepOutcome divide(String id, Handoff handoff, Driving state, long at) { + boolean folding = handoff.kind() == Handoff.Kind.COMBINE; + HandoffContext context = context(handoff, state); + HookResult answer = folding ? hooks.combine(context) : hooks.divide(context); + if (answer instanceof HookResult.Permanent) { + return advanced(id, HandoffState.DIVIDING, stepMachine(id, "dividePermanent", at)); + } + return afterHook(id, state, at, HandoffState.DIVIDING, answer, + folding ? "combineSuccess" : "divideSuccess", "attemptsExhausted"); } private StepOutcome transfer(String id, Handoff handoff, Driving state, long at) { @@ -344,11 +409,27 @@ private StepOutcome commit(String id, Handoff handoff, Driving state, long at) { if ("idle".equals(outcome.outcome())) { // MOVE-333: the commit deadline ran past the horizon, so the lease is spent and // MOVE-331 takes a fresh quiesce on the next step. + // An idle outcome here is the spent lease where the machine says so, and a + // concurrent abort otherwise, which is not this event. + if ("quiesceExpired".equals(outcome.reason())) { + report("quiesce_expired", Severity.ERROR, Map.of("handoff", id, + "shard", handoff.shard(), + "leaseMillis", Long.toString(handoff.leaseMillis()), + "marginMillis", Long.toString(policy.quiesceLeaseMarginMillis()))); + } synchronized (machineLock) { machine.clearQuiesce(id); } return new StepOutcome.Progressed(HandoffId.of(id), 0); } + // MOVE-311: the window is the interval the shard was quiesced for, from the reading + // `MOVE-332` took before the commit to the reading this step was driven with. + report("cutover_committed", Severity.INFO, Map.of("shard", handoff.shard(), + "source", handoff.source().asText(), + "destination", handoff.destination().asText(), + "windowMillis", Long.toString( + handoff.quiesceInstant().isPresent() + ? at - handoff.quiesceInstant().getAsLong() : 0L))); return advanced(id, handoff.state(), outcome); } @@ -465,9 +546,9 @@ private StepOutcome advanced(String id, /** The context one hook call reads, under {@code MOVE-111}. */ private HandoffContext context(Handoff handoff, Driving state) { - return new HandoffContext(ShardId.of(handoff.shard()), source.topologyId(), - handoff.fromEpoch(), handoff.toEpoch(), handoff.source(), handoff.destination(), - state.attempts + 1, policy.stepDeadlineMillis()); + return new HandoffContext(ShardId.of(handoff.shard()), ShardId.of(handoff.sourceShard()), + source.topologyId(), handoff.fromEpoch(), handoff.toEpoch(), handoff.source(), + handoff.destination(), state.attempts + 1, policy.stepDeadlineMillis()); } @Override @@ -520,7 +601,7 @@ public ReobserveOutcome reobserve(HandoffId id, MonotonicClock clock) { // MOVE-233 re-observes a terminal handoff; a live one is advanced by `step`. return new ReobserveOutcome.Refused("the handoff is not terminal"); } - return switch (hooks.observe(context(handoff, state))) { + ReobserveOutcome outcome = switch (hooks.observe(context(handoff, state))) { case ObserveResult.Observed observed -> { resume(id.value(), handoff, observed.observation()); yield new ReobserveOutcome.Resumed(id, @@ -529,6 +610,15 @@ public ReobserveOutcome reobserve(HandoffId id, MonotonicClock clock) { case ObserveResult.Unavailable unavailable -> new ReobserveOutcome.Unresolved(); case ObserveResult.Undetermined undetermined -> new ReobserveOutcome.Unresolved(); }; + // OBS-020: `answer` is what the hook established, and `resumedState` the state the + // handoff is in afterwards, which is unchanged where nothing was established. + report("reobserved", Severity.INFO, Map.of("handoff", id.value(), + "shard", handoff.shard(), + "answer", outcome instanceof ReobserveOutcome.Resumed ? "observed" + : outcome instanceof ReobserveOutcome.Unresolved ? "unresolved" + : "refused", + "resumedState", machine.handoff(id.value()).state().spelling())); + return outcome; } finally { state.claimed.set(false); } @@ -603,8 +693,15 @@ public RebaseReport rebase(TopologySnapshot to) { var report = machine.rebase(newer.epoch(), replicaSets); target = newer; delta = com.codeheadsystems.sharder.topology.OwnershipDelta.between(source, newer); - return new RebaseReport(report.fromEpoch(), report.toEpoch(), + RebaseReport answer = new RebaseReport(report.fromEpoch(), report.toEpoch(), ids(report.rebased()), ids(report.aborted()), ids(report.unchanged())); + report("rebased", Severity.INFO, Map.of( + "fromEpoch", Long.toString(answer.fromEpoch()), + "toEpoch", Long.toString(answer.toEpoch()), + "rebased", Integer.toString(answer.rebased().size()), + "aborted", Integer.toString(answer.aborted().size()), + "unchanged", Integer.toString(answer.unchanged().size()))); + return answer; } } finally { claims.forEach(claim -> claim.claimed.set(false)); @@ -678,12 +775,25 @@ public void onSnapshotInstalled(TopologySnapshot snapshot) { if (!(snapshot instanceof DocumentSnapshot installed)) { throw new InvalidArgumentException("the snapshot was not produced by this library"); } + com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan.InstallOutcome outcome; synchronized (machineLock) { // MOVE-092: the mark records the installed snapshot and nothing else. No hook is // called, no preference list is evaluated, and no delta is computed. - machine.onSnapshotInstalled(installed.topologyId(), installed.epoch(), + outcome = machine.onSnapshotInstalled(installed.topologyId(), installed.epoch(), source.topologyId()); } + if (outcome.superseded()) { + report("superseded", Severity.WARNING, Map.of( + "installedEpoch", Long.toString(installed.epoch()), + "abortedCount", Integer.toString(outcome.aborted().size()), + "finishingCount", Integer.toString(outcome.finishing().size()))); + } + if (outcome.rebasePending() != null) { + // MOVE-091: the plan is marked and the integrator decides whether to rebase it. + report("rebase_pending", Severity.WARNING, Map.of( + "installedEpoch", Long.toString(installed.epoch()), + "targetEpoch", Long.toString(outcome.rebasePending()))); + } } private static List ids(List named) { diff --git a/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java b/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java index 31713d3..5813ec0 100644 --- a/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java +++ b/ports/java/src/main/java/com/codeheadsystems/sharder/topology/OwnershipDelta.java @@ -11,9 +11,15 @@ * The shards whose replica sets differ between two snapshots, under {@code TOPO-211}. * *

The delta is computed on demand rather than at installation, and it reads the replica prefix - * of each shard rather than the whole preference list. A shard the later snapshot does not - * enumerate is absent from the delta, and a shard the earlier one did not enumerate has an empty - * before set. + * of each shard rather than the whole preference list. Its entries come in the two groups + * {@code TOPO-213} fixes: first the shards the later snapshot enumerates, in that snapshot's + * order, then the shards only the earlier one enumerates. A shard the earlier snapshot did not + * enumerate has an empty before set, and one the later snapshot does not enumerate has an empty + * after set. + * + *

A delta joins the two snapshots on the shard identifier, so it answers which shards changed + * owner and not where a new shard's contents are. Where an epoch changes which shards exist, that + * second question is answered by {@code HandoffCoordinator.lineage}. */ public record OwnershipDelta(List changes) { diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java b/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java index 0833e65..20f6306 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/api/MigrationTest.java @@ -15,7 +15,9 @@ import com.codeheadsystems.sharder.migrate.HookDeclaration; import com.codeheadsystems.sharder.migrate.HookResult; import com.codeheadsystems.sharder.migrate.MigrationPlan; +import com.codeheadsystems.sharder.migrate.HandoffCoordinator; import com.codeheadsystems.sharder.migrate.MigrationPolicy; +import com.codeheadsystems.sharder.observe.Event; import com.codeheadsystems.sharder.NodeId; import com.codeheadsystems.sharder.ShardId; import com.codeheadsystems.sharder.error.InvalidArgumentException; @@ -83,6 +85,60 @@ private static byte[] slots(long epoch, String... owners) { return json.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); } + /** + * {@code OBS-020}: the `migration.` events a coordinator reports, under {@code adr/0092}. + * + *

The suite asserts no emitted event for any surface, so this is where the port's own + * emission is checked. It drives one handoff from `planned` to `complete` and asserts the + * events that run produces, their severities, and the common members of {@code OBS-021}. + */ + @Test + void aCoordinatorReportsTheEventsOfItsSurface() { + Pair pair = snapshots(slots(1, "a", "b"), slots(2, "a", "c")); + List events = new java.util.concurrent.CopyOnWriteArrayList<>(); + RouterConfig config = RouterConfig.builder() + .provider(new InMemoryTopologyProvider(slots(1, "a", "b"))) + .clock(CLOCK) + .eventSink(events::add) + .build(); + HandoffCoordinator coordinator = Sharder.coordinator(config); + + coordinator.lineage(pair.from(), pair.to()); + MigrationPlan plan = coordinator.plan(pair.from(), pair.to(), new RecordingHooks(), + MigrationPolicy.defaults()); + for (int step = 0; step < 12; step++) { + if (plan.step(CLOCK) instanceof StepOutcome.Settled) { + break; + } + } + + List names = events.stream().map(Event::name).distinct().toList(); + assertThat(names) + .contains("sharder.migration.lineage", "sharder.migration.planned", + "sharder.migration.state_changed", "sharder.migration.cutover_committed"); + // OBS-021: every event carries the topology and epoch in force and a severity. + assertThat(events).allSatisfy(event -> { + assertThat(event.topologyId()).isEqualTo("slots"); + assertThat(event.epoch()).isEqualTo(2L); + assertThat(event.severity()).isNotNull(); + assertThat(event.payload()).isNotEmpty(); + }); + // Every payload member the inventory names for an emitted event is carried. + Event planned = events.stream() + .filter(event -> event.name().equals("sharder.migration.planned")) + .findFirst().orElseThrow(); + assertThat(planned.payload()).containsKeys("handoffCount", "policy"); + Event lineage = events.stream() + .filter(event -> event.name().equals("sharder.migration.lineage")) + .findFirst().orElseThrow(); + assertThat(lineage.payload()).containsKeys("unchanged", "moved", "divided", "merged", + "fresh", "split", "folded", "vacated"); + Event changed = events.stream() + .filter(event -> event.name().equals("sharder.migration.state_changed")) + .findFirst().orElseThrow(); + assertThat(changed.payload()).containsKeys("handoff", "shard", "from", "to", "trigger"); + } + @Test void oneHandoffRunsFromPlannedToComplete() { Pair pair = snapshots(slots(1, "a", "b"), slots(2, "a", "c")); @@ -171,7 +227,7 @@ void aMismatchedVerificationFailsTheHandoff() { void hooksThatDeclareNoVerificationReachCleanupFromTheRecord() { Pair pair = snapshots(slots(1, "a", "b"), slots(2, "a", "c")); RecordingHooks hooks = new RecordingHooks() - .declaring(new HookDeclaration("rows", true, false)); + .declaring(new HookDeclaration("rows", true, false, false)); MigrationPlan plan = Sharder.coordinator() .plan(pair.from(), pair.to(), hooks, MigrationPolicy.defaults()); for (int step = 0; step < 12; step++) { diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java index e1ff8c9..fb1e348 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ConformanceSuite.java @@ -143,6 +143,8 @@ private void runCase(String kind, JsonObject testCase, PlaceVectors place, case "observabilityInventory" -> core.observabilityInventory(testCase); case "publicationEvents" -> core.publicationEvents(testCase); case "ownershipDelta" -> core.ownershipDelta(testCase); + case "lineage" -> core.lineage(testCase); + case "planConstruction" -> core.planConstruction(testCase); case "propertyWitness" -> core.propertyWitness(testCase); case "scale" -> core.scale(testCase); case "readAffinity" -> core.readAffinity(testCase); diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java index 56843c8..15124a4 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/CoreVectors.java @@ -1,6 +1,7 @@ package com.codeheadsystems.sharder.conformance; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.codeheadsystems.sharder.Digest; import com.codeheadsystems.sharder.NodeId; @@ -13,9 +14,18 @@ import com.codeheadsystems.sharder.core.internal.observe.Observability; import com.codeheadsystems.sharder.core.internal.observe.PublicationEvents; import com.codeheadsystems.sharder.core.internal.observe.SkewDetection; +import com.codeheadsystems.sharder.core.internal.migrate.Handoff; +import com.codeheadsystems.sharder.core.internal.migrate.MigrationPlan; +import com.codeheadsystems.sharder.core.internal.placement.ShardExtents; import com.codeheadsystems.sharder.core.internal.route.OwnershipDelta; import com.codeheadsystems.sharder.core.internal.route.PlacementEngine; +import com.codeheadsystems.sharder.ShardId; +import com.codeheadsystems.sharder.core.Sharder; import com.codeheadsystems.sharder.error.ErrorCode; +import com.codeheadsystems.sharder.error.PlanRefusedException; +import com.codeheadsystems.sharder.migrate.HandoffCoordinator; +import com.codeheadsystems.sharder.migrate.ShardLineage; +import com.codeheadsystems.sharder.topology.TopologySnapshot; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HexFormat; @@ -289,6 +299,88 @@ void ownershipDelta(JsonObject testCase) { } } + /** {@code lineage}: the classification of two snapshots' extents, under {@code LIN-021}. */ + void lineage(JsonObject testCase) { + // LIN-031 requires the lineage to be an operation the integrator calls, so the harness + // reaches it through the exported surface rather than through the class behind it. A port + // that computed a lineage it could not expose would pass the one and fail the other. + HandoffCoordinator coordinator = Sharder.coordinator(); + TopologySnapshot from = snapshotOf(testCase, "before"); + TopologySnapshot to = snapshotOf(testCase, "after"); + JsonObject expect = testCase.object("expect"); + if (expect.find("lineageComputed").isPresent()) { + JsonObject condition = expect.object("condition"); + ErrorCode code = ErrorCode.ofName(condition.text("name")); + assertThat(code.code()).as("condition code").isEqualTo(condition.get("code").asInt()); + assertThatThrownBy(() -> coordinator.lineage(from, to)) + .as("refusal").isInstanceOf(PlanRefusedException.class); + try { + coordinator.lineage(from, to); + } catch (PlanRefusedException refusal) { + assertThat(refusal.reason().spelling()).as("cause") + .isEqualTo(condition.text("cause")); + assertThat(code.causes()).as("cause is in the closed set") + .contains(condition.text("cause")); + } + return; + } + List entries = coordinator.lineage(from, to).entries(); + List expected = expect.array("lineage").elements(); + assertThat(entries).as("lineage").hasSize(expected.size()); + for (int index = 0; index < expected.size(); index++) { + JsonObject entry = expected.get(index).asObject(); + ShardLineage.Entry actual = entries.get(index); + assertThat(actual.shard().asText()).as("lineage[%d].shard", index) + .isEqualTo(entry.text("shard")); + assertThat(actual.lineage().spelling()).as("lineage[%d].class", index) + .isEqualTo(entry.text("class")); + assertThat(actual.parents().stream().map(ShardId::asText).toList()) + .as("lineage[%d].parents", index).isEqualTo(entry.array("parents").texts()); + } + } + + /** The exported snapshot type over a document a vector named, for a coordinator call. */ + private TopologySnapshot snapshotOf(JsonObject testCase, String member) { + JsonObject document = source.readObject(testCase.text(member)); + return new com.codeheadsystems.sharder.core.internal.snapshot.DocumentSnapshot( + new PlacementEngine(com.codeheadsystems.sharder.core.internal.document + .TopologyDocument.parse(document)), + Digests.of(JcsWriter.canonicalise(document)), java.util.OptionalLong.empty()); + } + + /** {@code planConstruction}: how a plan derives a handoff from a lineage, {@code LIN-041}. */ + void planConstruction(JsonObject testCase) { + PlacementEngine before = engine(testCase, "before"); + PlacementEngine after = engine(testCase, "after"); + MigrationPlan plan = MigrationPlan.of(before, after, 0L); + List expected = testCase.object("expect").array("handoffs").elements(); + assertThat(plan.handoffs()).as("handoff count").hasSize(expected.size()); + for (int index = 0; index < expected.size(); index++) { + JsonObject entry = expected.get(index).asObject(); + Handoff handoff = plan.handoff(entry.text("id")); + assertThat(handoff).as("handoffs[%d] named %s", index, entry.text("id")).isNotNull(); + assertThat(handoff.shard()).as("handoffs[%d].shard", index) + .isEqualTo(entry.text("shard")); + assertThat(handoff.kind().spelling()).as("handoffs[%d].kind", index) + .isEqualTo(entry.text("kind")); + assertThat(handoff.sourceShard()).as("handoffs[%d].sourceShard", index) + .isEqualTo(entry.text("sourceShard")); + assertThat(handoff.source().asText()).as("handoffs[%d].source", index) + .isEqualTo(entry.text("source")); + assertThat(handoff.destination().asText()).as("handoffs[%d].destination", index) + .isEqualTo(entry.text("destination")); + if (handoff.kind().local()) { + // LIN-051: a local step moves nothing between nodes, so it names one node twice. + assertThat(handoff.source()).as("handoffs[%d] is local", index) + .isEqualTo(handoff.destination()); + } else { + // LIN-042: a node cannot both hold the contents and be the node they move to. + assertThat(handoff.source()).as("handoffs[%d] source is not the destination", index) + .isNotEqualTo(handoff.destination()); + } + } + } + /** {@code propertyWitness}: a sampled bound over the sample {@code PROP-006} fixes. */ void propertyWitness(JsonObject testCase) { JsonObject expect = testCase.object("expect"); diff --git a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java index 9390b92..023cfc0 100644 --- a/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java +++ b/ports/java/src/test/java/com/codeheadsystems/sharder/conformance/ScenarioRunner.java @@ -42,7 +42,17 @@ final class ScenarioRunner { /** One scenario's steps, replayed in order, answering the count this driver skipped. */ int run(String path) { JsonObject scenario = source.readObject(path); - TopologyLoader loader = new TopologyLoader(); + // TOPO-061 and TOPO-071: a scenario states the identifier and the epoch floor its process + // was configured with, both of which the pipeline checks before it installs anything. + JsonObject loaderSetup = scenario.object("setup").find("loader") + .map(JsonValue::asObject).orElse(null); + TopologyLoader loader = loaderSetup == null + ? new TopologyLoader() + : new TopologyLoader(java.util.Map.of(), + loaderSetup.find("expectedTopologyId").map(JsonValue::asText), + loaderSetup.find("minEpoch") + .map(value -> java.util.OptionalLong.of(value.asLong())) + .orElseGet(java.util.OptionalLong::empty)); SlidingWindowHealthView[] health = {new SlidingWindowHealthView(HealthSettings.defaults())}; RetryBudget budget = RetryBudget.defaults(); int skipped = 0; @@ -62,7 +72,19 @@ int run(String path) { new java.util.ArrayList<>(); for (JsonValue row : spec.array("handoffs").elements()) { JsonObject entry = row.asObject(); - named.add(MigrationPlan.handoffOf(entry.text("id"), entry.text("shard"), + String shard = entry.text("shard"); + String sourceShard = entry.find("sourceShard") + .map(JsonValue::asText).orElse(shard); + com.codeheadsystems.sharder.core.internal.migrate.Handoff.Kind kind = + switch (entry.find("kind").map(JsonValue::asText).orElse("handoff")) { + case "divide" -> com.codeheadsystems.sharder.core.internal.migrate + .Handoff.Kind.DIVIDE; + case "combine" -> com.codeheadsystems.sharder.core.internal.migrate + .Handoff.Kind.COMBINE; + default -> com.codeheadsystems.sharder.core.internal.migrate + .Handoff.Kind.HANDOFF; + }; + named.add(MigrationPlan.handoffOf(entry.text("id"), shard, sourceShard, kind, NodeId.of(entry.text("source")), NodeId.of(entry.text("destination")), spec.get("fromEpoch").asLong(), spec.get("toEpoch").asLong())); }