diff --git a/test/smoke/runner.py b/test/smoke/runner.py
index 3eaf0193563..1c339f9d491 100644
--- a/test/smoke/runner.py
+++ b/test/smoke/runner.py
@@ -18,8 +18,8 @@
REQUIRED = {(client, "moq-dev-rs") for client in OWN}
-def matrices(registry):
- """Keep published images and partition the pairs around our relay and clients."""
+def matrices(registry, missing=frozenset()):
+ """Keep the published images not in `missing` and partition the pairs around our relay and clients."""
registry = copy.deepcopy(registry)
implementations = registry["implementations"]
relay = implementations["moq-dev-rs"]["roles"]["relay"]["docker"]
@@ -33,16 +33,14 @@ def matrices(registry):
continue
image = docker["image"]
# Local adapter tags need a build even when their base is published.
- local = bool(docker.get("build")) and "/" not in image
- inventory.append(
- {
- "implementation": name,
- "role": role,
- "image": image,
- "status": "not run: local build only" if local else "selected",
- }
- )
- if local:
+ if docker.get("build") and "/" not in image:
+ status = "not run: local build only"
+ elif image in missing:
+ status = "not run: image unavailable"
+ else:
+ status = "selected"
+ inventory.append({"implementation": name, "role": role, "image": image, "status": status})
+ if status != "selected":
del implementation["roles"][role]["docker"]
if any("docker" not in implementations[name]["roles"]["client"] for name in OWN):
raise ValueError("the registry must provide both moq-dev client images")
@@ -59,25 +57,32 @@ def matrices(registry):
return (to_relay, from_clients), inventory
-def expected_pairs(registry):
+def modes(docker):
+ """The runner's modes for a docker relay: one per declared endpoint, else plain docker."""
+ return [f"docker-{endpoint['transport']}" for endpoint in docker.get("endpoints", [])] or ["docker"]
+
+
+def expected_runs(registry):
+ """Every (client, relay, mode) run the runner should report for this registry."""
implementations = registry["implementations"]
return {
- (client, relay)
+ (client, relay, mode)
for client, client_info in implementations.items()
if client_info.get("roles", {}).get("client", {}).get("docker")
for relay, relay_info in implementations.items()
if relay_info.get("roles", {}).get("relay", {}).get("docker")
and set(client_info["draft_versions"]) & set(relay_info["draft_versions"])
+ for mode in modes(relay_info["roles"]["relay"]["docker"])
}
def validate_summary(summary, expected):
runs = summary["runs"]
- actual = {(run["client"], run["relay"]) for run in runs}
+ actual = {(run["client"], run["relay"], run["mode"]) for run in runs}
if actual != expected or len(runs) != len(expected):
raise ValueError(f"incomplete runner coverage: expected {sorted(expected)}, got {sorted(actual)}")
- if any(run["mode"] != "docker" or run["status"] not in ("pass", "fail", "timeout") for run in runs):
- raise ValueError("unexpected mode, skipped image, or unknown verdict in runner summary")
+ if any(run["status"] not in ("pass", "fail", "timeout") for run in runs):
+ raise ValueError("skipped image or unknown verdict in runner summary")
def run(upstream, output):
@@ -85,18 +90,28 @@ def run(upstream, output):
raise ValueError(f"report destination already exists: {output}")
registry_file = upstream / "implementations.json"
original = registry_file.read_text()
- plans, inventory = matrices(json.loads(original))
+ registry = json.loads(original)
for command in (["docker", "info"], ["docker", "compose", "version"]):
subprocess.run(command, check=True)
+ unfiltered, inventory = matrices(registry)
+ own = {registry["implementations"][name]["roles"]["client"]["docker"]["image"] for name in OWN}
+ missing = set()
images = {entry["image"] for entry in inventory if entry["status"] == "selected"}
for image in sorted(images - {RELAY_IMAGE}):
# A registry blip would otherwise abort both matrices before any report.
for attempt in range(1, 4):
if not subprocess.run(["docker", "pull", image]).returncode:
break
- if attempt == 3:
+ if attempt < 3:
+ time.sleep(attempt * 2)
+ elif image in own:
raise SystemExit(f"failed to pull {image}")
- time.sleep(attempt * 2)
+ else:
+ # A third party's unpublished image only removes its optional pairs, listed under Not run.
+ missing.add(image)
+ if missing:
+ print(f"::warning::images unavailable, their pairs not run: {sorted(missing)}")
+ plans, inventory = matrices(registry, missing)
subprocess.run(["docker", "image", "inspect", RELAY_IMAGE], check=True)
subprocess.run(["make", "certs"], cwd=upstream, check=True)
results = upstream / "results"
@@ -107,8 +122,11 @@ def run(upstream, output):
try:
for index, plan in enumerate(plans):
registry_file.write_text(json.dumps(plan))
- expected = expected_pairs(plan)
+ expected = expected_runs(plan)
if not expected:
+ # Unpublished images can empty the optional matrix; the registry alone must not.
+ if expected_runs(unfiltered[index]):
+ continue
raise ValueError(f"runner matrix {index} has no compatible pairs")
# The runner exits nonzero on any red pair; the validated summary decides instead.
subprocess.run(["bash", "run-interop-tests.sh", "--docker-only"], cwd=upstream)
@@ -137,7 +155,7 @@ def run(upstream, output):
if skipped:
rows = "".join(
f"
{html.escape(entry['implementation'])} {entry['role']}: "
- f"{html.escape(entry['image'])} (local build only)"
+ f"{html.escape(entry['image'])} ({html.escape(entry['status'].removeprefix('not run: '))})"
for entry in skipped
)
index_file = results / "index.html"
@@ -150,12 +168,13 @@ def run(upstream, output):
)
# Fail only after the report shows every pair. A required pair the registry
# no longer pairs fails too, rather than passing unchecked.
- status = {(run["client"], run["relay"]): run["status"] for run in combined["runs"]}
- optional = sorted(pair for pair, verdict in status.items() if verdict != "pass" and pair not in REQUIRED)
+ red = {(run["client"], run["relay"], run["mode"]) for run in combined["runs"] if run["status"] != "pass"}
+ optional = sorted(run for run in red if run[:2] not in REQUIRED)
if optional:
# A workflow annotation, so a green job still shows them.
- print(f"::warning::optional pairs not passing (reported only): {optional}")
- failed = sorted(pair for pair in REQUIRED if status.get(pair) != "pass")
+ print(f"::warning::optional runs not passing (reported only): {optional}")
+ ran = {(run["client"], run["relay"]) for run in combined["runs"]}
+ failed = sorted(pair for pair in REQUIRED if pair not in ran or any(run[:2] == pair for run in red))
if failed:
raise SystemExit(f"required pairs failed: {failed}; see the report")
diff --git a/test/smoke/runner.test.py b/test/smoke/runner.test.py
index fd1c751834d..0cdb775d966 100644
--- a/test/smoke/runner.test.py
+++ b/test/smoke/runner.test.py
@@ -21,6 +21,10 @@ def implementation(**roles):
}
+def pairs(plan):
+ return {(client, relay) for client, relay, _ in runner.expected_runs(plan)}
+
+
class Runner(unittest.TestCase):
def setUp(self):
self.registry = {
@@ -35,10 +39,10 @@ def setUp(self):
def test_new_published_implementations_join_both_directions_once(self):
plans, inventory = runner.matrices(self.registry)
- pairs = [runner.expected_pairs(plan) for plan in plans]
- self.assertEqual(pairs[0], {("moq-dev-rs", "moq-dev-rs"), ("moq-dev-js", "moq-dev-rs"), ("new", "moq-dev-rs")})
- self.assertEqual(pairs[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")})
- self.assertFalse(pairs[0] & pairs[1])
+ matrix = [pairs(plan) for plan in plans]
+ self.assertEqual(matrix[0], {("moq-dev-rs", "moq-dev-rs"), ("moq-dev-js", "moq-dev-rs"), ("new", "moq-dev-rs")})
+ self.assertEqual(matrix[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")})
+ self.assertFalse(matrix[0] & matrix[1])
self.assertEqual(
next(entry["status"] for entry in inventory if entry["implementation"] == "adapter"),
"not run: local build only",
@@ -50,80 +54,141 @@ def test_new_published_implementations_join_both_directions_once(self):
def test_published_image_with_source_recipe_is_still_selected(self):
self.registry["implementations"]["new"]["roles"]["relay"]["docker"]["build"] = {"context": "."}
plans, _ = runner.matrices(self.registry)
- self.assertIn(("moq-dev-rs", "new"), runner.expected_pairs(plans[1]))
+ self.assertIn(("moq-dev-rs", "new"), pairs(plans[1]))
def test_missing_skipped_duplicate_or_unexpected_cells_fail(self):
cell = {"client": "a", "relay": "b", "mode": "docker", "status": "fail"}
- runner.validate_summary({"runs": [cell]}, {("a", "b")})
- for cells in ([], [cell, cell], [dict(cell, status="skip")], [dict(cell, relay="c")]):
+ runner.validate_summary({"runs": [cell]}, {("a", "b", "docker")})
+ wrong = [dict(cell, status="skip"), dict(cell, relay="c"), dict(cell, mode="docker-quic")]
+ for cells in ([], [cell, cell], *([bad] for bad in wrong)):
with self.subTest(cells=cells), self.assertRaises(ValueError):
- runner.validate_summary({"runs": cells}, {("a", "b")})
+ runner.validate_summary({"runs": cells}, {("a", "b", "docker")})
+
+ def test_relay_endpoints_each_run_and_gate_their_pair(self):
+ self.registry["implementations"]["new"]["roles"]["relay"]["docker"]["endpoints"] = [
+ {"transport": "quic", "url": "moqt://relay:4443"},
+ {"transport": "webtransport", "url": "https://relay:4443"},
+ ]
+ plans, _ = runner.matrices(self.registry)
+ self.assertEqual(
+ {run for run in runner.expected_runs(plans[1]) if run[0] == "moq-dev-rs"},
+ {("moq-dev-rs", "new", "docker-quic"), ("moq-dev-rs", "new", "docker-webtransport")},
+ )
+ with patch("builtins.print") as report:
+ failure = self.run_matrices(lambda client, relay, mode: "fail" if mode == "docker-quic" else "pass")
+ self.assertIsNone(failure)
+ self.assertIn("('moq-dev-js', 'new', 'docker-quic')", report.call_args.args[0])
+
+ def run_matrices(self, status, unpublished=frozenset()):
+ """Run both matrices against a fake runner that gives each pair `status(client, relay)`.
- def run_matrices(self, status):
- """Run both matrices against a fake runner that gives each pair `status(client, relay)`."""
+ Pulling an image in `unpublished` always fails.
+ """
with tempfile.TemporaryDirectory() as temporary:
upstream = Path(temporary)
registry = upstream / "implementations.json"
original = json.dumps(self.registry)
registry.write_text(original)
matrices = []
+ reported = []
def command(args, **kwargs):
results = upstream / "results"
if args[:2] == ["bash", "run-interop-tests.sh"]:
- pairs = runner.expected_pairs(json.loads(registry.read_text()))
+ plan = json.loads(registry.read_text())
result = results / str(len(matrices))
result.mkdir(parents=True)
runs = [
- {"client": client, "relay": relay, "mode": "docker", "status": status(client, relay)}
- for client, relay in pairs
+ {"client": client, "relay": relay, "mode": mode, "status": status(client, relay, mode)}
+ for client, relay, mode in runner.expected_runs(plan)
]
(result / "summary.json").write_text(json.dumps({"runs": runs}))
- matrices.append(pairs)
+ matrices.append(pairs(plan))
+ reported.extend(runs)
failed = any(run["status"] != "pass" for run in runs)
return subprocess.CompletedProcess(args, int(failed))
+ if args[:2] == ["docker", "pull"]:
+ return subprocess.CompletedProcess(args, int(args[2] in unpublished))
if args[:2] == ["bash", "generate-report.sh"]:
(results / "index.html").write_text("")
(results / "nightly" / "report.html").write_text("")
return subprocess.CompletedProcess(args, 0)
failure = None
- with patch.object(runner.subprocess, "run", side_effect=command):
+ self.matrices = matrices
+ with patch.object(runner.subprocess, "run", side_effect=command), patch.object(runner.time, "sleep"):
try:
runner.run(upstream, upstream / "report")
except SystemExit as error:
failure = error
- self.assertEqual(len(matrices), 2)
self.assertEqual(registry.read_text(), original)
runs = json.loads((upstream / "report" / "nightly" / "summary.json").read_text())["runs"]
- self.assertEqual(len(runs), sum(len(pairs) for pairs in matrices))
- self.assertIn("local-adapter", (upstream / "report" / "index.html").read_text())
+ self.assertEqual(runs, reported)
+ self.index = (upstream / "report" / "index.html").read_text()
+ self.assertIn("local-adapter", self.index)
return failure
def test_failed_required_pair_reports_every_matrix_then_fails(self):
required = ("moq-dev-js", "moq-dev-rs")
- failure = self.run_matrices(lambda client, relay: "timeout" if (client, relay) == required else "pass")
+ failure = self.run_matrices(lambda client, relay, mode: "timeout" if (client, relay) == required else "pass")
+ self.assertEqual(len(self.matrices), 2)
self.assertIn(str(required), str(failure))
def test_failed_optional_pairs_are_reported_but_pass(self):
- def status(client, relay):
+ def status(client, relay, mode):
return "pass" if (client, relay) in runner.REQUIRED else "fail"
with patch("builtins.print") as report:
failure = self.run_matrices(status)
self.assertIsNone(failure)
- self.assertIn("('new', 'moq-dev-rs')", report.call_args.args[0])
+ self.assertIn("('new', 'moq-dev-rs', 'docker')", report.call_args.args[0])
+
+ def test_unpublished_third_party_image_drops_only_its_pairs(self):
+ with patch("builtins.print") as report:
+ failure = self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/client"})
+ self.assertIsNone(failure)
+ self.assertEqual(self.matrices[0], runner.REQUIRED)
+ self.assertEqual(self.matrices[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")})
+ self.assertIn("docker.io/new/client", report.call_args.args[0])
+ self.assertIn("docker.io/new/client (image unavailable)", self.index)
+
+ def test_unpublished_last_third_party_relay_skips_the_optional_matrix(self):
+ with patch("builtins.print"):
+ failure = self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/relay"})
+ self.assertIsNone(failure)
+ self.assertEqual(self.matrices, [runner.REQUIRED | {("new", "moq-dev-rs")}])
+
+ def test_matrix_the_registry_leaves_empty_fails_despite_unrelated_missing_image(self):
+ del self.registry["implementations"]["new"]["roles"]["relay"]
+ with patch("builtins.print"), self.assertRaisesRegex(ValueError, "matrix 1 has no compatible pairs"):
+ self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/client"})
+
+ def test_unpublished_own_client_image_fails_before_any_matrix(self):
+ def command(args, **kwargs):
+ self.assertNotEqual(args[:2], ["bash", "run-interop-tests.sh"])
+ return subprocess.CompletedProcess(args, int(args == ["docker", "pull", "ghcr.io/ours/js"]))
+
+ with tempfile.TemporaryDirectory() as temporary:
+ upstream = Path(temporary)
+ (upstream / "implementations.json").write_text(json.dumps(self.registry))
+ with (
+ patch.object(runner.subprocess, "run", side_effect=command),
+ patch.object(runner.time, "sleep"),
+ self.assertRaises(SystemExit) as failure,
+ ):
+ runner.run(upstream, upstream / "report")
+ self.assertIn("ghcr.io/ours/js", str(failure.exception))
def test_required_pair_the_registry_drops_fails(self):
self.registry["implementations"]["moq-dev-js"]["draft_versions"] = ["draft-22"]
self.registry["implementations"]["new"]["draft_versions"] = ["draft-18", "draft-22"]
- self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay: "pass")))
+ self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay, mode: "pass")))
def test_drafts_without_overlap_have_no_pair(self):
self.registry["implementations"]["new"]["draft_versions"] = ["draft-22"]
plans, _ = runner.matrices(self.registry)
- self.assertNotIn(("new", "moq-dev-rs"), runner.expected_pairs(plans[0]))
- self.assertFalse(runner.expected_pairs(plans[1]))
+ self.assertNotIn(("new", "moq-dev-rs"), pairs(plans[0]))
+ self.assertFalse(pairs(plans[1]))
if __name__ == "__main__":