diff --git a/test/smoke/runner.py b/test/smoke/runner.py index 3eaf0193563..1c339f9d491 100644 --- a/test/smoke/runner.py +++ b/test/smoke/runner.py @@ -18,8 +18,8 @@ REQUIRED = {(client, "moq-dev-rs") for client in OWN} -def matrices(registry): - """Keep published images and partition the pairs around our relay and clients.""" +def matrices(registry, missing=frozenset()): + """Keep the published images not in `missing` and partition the pairs around our relay and clients.""" registry = copy.deepcopy(registry) implementations = registry["implementations"] relay = implementations["moq-dev-rs"]["roles"]["relay"]["docker"] @@ -33,16 +33,14 @@ def matrices(registry): continue image = docker["image"] # Local adapter tags need a build even when their base is published. - local = bool(docker.get("build")) and "/" not in image - inventory.append( - { - "implementation": name, - "role": role, - "image": image, - "status": "not run: local build only" if local else "selected", - } - ) - if local: + if docker.get("build") and "/" not in image: + status = "not run: local build only" + elif image in missing: + status = "not run: image unavailable" + else: + status = "selected" + inventory.append({"implementation": name, "role": role, "image": image, "status": status}) + if status != "selected": del implementation["roles"][role]["docker"] if any("docker" not in implementations[name]["roles"]["client"] for name in OWN): raise ValueError("the registry must provide both moq-dev client images") @@ -59,25 +57,32 @@ def matrices(registry): return (to_relay, from_clients), inventory -def expected_pairs(registry): +def modes(docker): + """The runner's modes for a docker relay: one per declared endpoint, else plain docker.""" + return [f"docker-{endpoint['transport']}" for endpoint in docker.get("endpoints", [])] or ["docker"] + + +def expected_runs(registry): + """Every (client, relay, mode) run the runner should report for this registry.""" implementations = registry["implementations"] return { - (client, relay) + (client, relay, mode) for client, client_info in implementations.items() if client_info.get("roles", {}).get("client", {}).get("docker") for relay, relay_info in implementations.items() if relay_info.get("roles", {}).get("relay", {}).get("docker") and set(client_info["draft_versions"]) & set(relay_info["draft_versions"]) + for mode in modes(relay_info["roles"]["relay"]["docker"]) } def validate_summary(summary, expected): runs = summary["runs"] - actual = {(run["client"], run["relay"]) for run in runs} + actual = {(run["client"], run["relay"], run["mode"]) for run in runs} if actual != expected or len(runs) != len(expected): raise ValueError(f"incomplete runner coverage: expected {sorted(expected)}, got {sorted(actual)}") - if any(run["mode"] != "docker" or run["status"] not in ("pass", "fail", "timeout") for run in runs): - raise ValueError("unexpected mode, skipped image, or unknown verdict in runner summary") + if any(run["status"] not in ("pass", "fail", "timeout") for run in runs): + raise ValueError("skipped image or unknown verdict in runner summary") def run(upstream, output): @@ -85,18 +90,28 @@ def run(upstream, output): raise ValueError(f"report destination already exists: {output}") registry_file = upstream / "implementations.json" original = registry_file.read_text() - plans, inventory = matrices(json.loads(original)) + registry = json.loads(original) for command in (["docker", "info"], ["docker", "compose", "version"]): subprocess.run(command, check=True) + unfiltered, inventory = matrices(registry) + own = {registry["implementations"][name]["roles"]["client"]["docker"]["image"] for name in OWN} + missing = set() images = {entry["image"] for entry in inventory if entry["status"] == "selected"} for image in sorted(images - {RELAY_IMAGE}): # A registry blip would otherwise abort both matrices before any report. for attempt in range(1, 4): if not subprocess.run(["docker", "pull", image]).returncode: break - if attempt == 3: + if attempt < 3: + time.sleep(attempt * 2) + elif image in own: raise SystemExit(f"failed to pull {image}") - time.sleep(attempt * 2) + else: + # A third party's unpublished image only removes its optional pairs, listed under Not run. + missing.add(image) + if missing: + print(f"::warning::images unavailable, their pairs not run: {sorted(missing)}") + plans, inventory = matrices(registry, missing) subprocess.run(["docker", "image", "inspect", RELAY_IMAGE], check=True) subprocess.run(["make", "certs"], cwd=upstream, check=True) results = upstream / "results" @@ -107,8 +122,11 @@ def run(upstream, output): try: for index, plan in enumerate(plans): registry_file.write_text(json.dumps(plan)) - expected = expected_pairs(plan) + expected = expected_runs(plan) if not expected: + # Unpublished images can empty the optional matrix; the registry alone must not. + if expected_runs(unfiltered[index]): + continue raise ValueError(f"runner matrix {index} has no compatible pairs") # The runner exits nonzero on any red pair; the validated summary decides instead. subprocess.run(["bash", "run-interop-tests.sh", "--docker-only"], cwd=upstream) @@ -137,7 +155,7 @@ def run(upstream, output): if skipped: rows = "".join( f"
  • {html.escape(entry['implementation'])} {entry['role']}: " - f"{html.escape(entry['image'])} (local build only)
  • " + f"{html.escape(entry['image'])} ({html.escape(entry['status'].removeprefix('not run: '))})" for entry in skipped ) index_file = results / "index.html" @@ -150,12 +168,13 @@ def run(upstream, output): ) # Fail only after the report shows every pair. A required pair the registry # no longer pairs fails too, rather than passing unchecked. - status = {(run["client"], run["relay"]): run["status"] for run in combined["runs"]} - optional = sorted(pair for pair, verdict in status.items() if verdict != "pass" and pair not in REQUIRED) + red = {(run["client"], run["relay"], run["mode"]) for run in combined["runs"] if run["status"] != "pass"} + optional = sorted(run for run in red if run[:2] not in REQUIRED) if optional: # A workflow annotation, so a green job still shows them. - print(f"::warning::optional pairs not passing (reported only): {optional}") - failed = sorted(pair for pair in REQUIRED if status.get(pair) != "pass") + print(f"::warning::optional runs not passing (reported only): {optional}") + ran = {(run["client"], run["relay"]) for run in combined["runs"]} + failed = sorted(pair for pair in REQUIRED if pair not in ran or any(run[:2] == pair for run in red)) if failed: raise SystemExit(f"required pairs failed: {failed}; see the report") diff --git a/test/smoke/runner.test.py b/test/smoke/runner.test.py index fd1c751834d..0cdb775d966 100644 --- a/test/smoke/runner.test.py +++ b/test/smoke/runner.test.py @@ -21,6 +21,10 @@ def implementation(**roles): } +def pairs(plan): + return {(client, relay) for client, relay, _ in runner.expected_runs(plan)} + + class Runner(unittest.TestCase): def setUp(self): self.registry = { @@ -35,10 +39,10 @@ def setUp(self): def test_new_published_implementations_join_both_directions_once(self): plans, inventory = runner.matrices(self.registry) - pairs = [runner.expected_pairs(plan) for plan in plans] - self.assertEqual(pairs[0], {("moq-dev-rs", "moq-dev-rs"), ("moq-dev-js", "moq-dev-rs"), ("new", "moq-dev-rs")}) - self.assertEqual(pairs[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")}) - self.assertFalse(pairs[0] & pairs[1]) + matrix = [pairs(plan) for plan in plans] + self.assertEqual(matrix[0], {("moq-dev-rs", "moq-dev-rs"), ("moq-dev-js", "moq-dev-rs"), ("new", "moq-dev-rs")}) + self.assertEqual(matrix[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")}) + self.assertFalse(matrix[0] & matrix[1]) self.assertEqual( next(entry["status"] for entry in inventory if entry["implementation"] == "adapter"), "not run: local build only", @@ -50,80 +54,141 @@ def test_new_published_implementations_join_both_directions_once(self): def test_published_image_with_source_recipe_is_still_selected(self): self.registry["implementations"]["new"]["roles"]["relay"]["docker"]["build"] = {"context": "."} plans, _ = runner.matrices(self.registry) - self.assertIn(("moq-dev-rs", "new"), runner.expected_pairs(plans[1])) + self.assertIn(("moq-dev-rs", "new"), pairs(plans[1])) def test_missing_skipped_duplicate_or_unexpected_cells_fail(self): cell = {"client": "a", "relay": "b", "mode": "docker", "status": "fail"} - runner.validate_summary({"runs": [cell]}, {("a", "b")}) - for cells in ([], [cell, cell], [dict(cell, status="skip")], [dict(cell, relay="c")]): + runner.validate_summary({"runs": [cell]}, {("a", "b", "docker")}) + wrong = [dict(cell, status="skip"), dict(cell, relay="c"), dict(cell, mode="docker-quic")] + for cells in ([], [cell, cell], *([bad] for bad in wrong)): with self.subTest(cells=cells), self.assertRaises(ValueError): - runner.validate_summary({"runs": cells}, {("a", "b")}) + runner.validate_summary({"runs": cells}, {("a", "b", "docker")}) + + def test_relay_endpoints_each_run_and_gate_their_pair(self): + self.registry["implementations"]["new"]["roles"]["relay"]["docker"]["endpoints"] = [ + {"transport": "quic", "url": "moqt://relay:4443"}, + {"transport": "webtransport", "url": "https://relay:4443"}, + ] + plans, _ = runner.matrices(self.registry) + self.assertEqual( + {run for run in runner.expected_runs(plans[1]) if run[0] == "moq-dev-rs"}, + {("moq-dev-rs", "new", "docker-quic"), ("moq-dev-rs", "new", "docker-webtransport")}, + ) + with patch("builtins.print") as report: + failure = self.run_matrices(lambda client, relay, mode: "fail" if mode == "docker-quic" else "pass") + self.assertIsNone(failure) + self.assertIn("('moq-dev-js', 'new', 'docker-quic')", report.call_args.args[0]) + + def run_matrices(self, status, unpublished=frozenset()): + """Run both matrices against a fake runner that gives each pair `status(client, relay)`. - def run_matrices(self, status): - """Run both matrices against a fake runner that gives each pair `status(client, relay)`.""" + Pulling an image in `unpublished` always fails. + """ with tempfile.TemporaryDirectory() as temporary: upstream = Path(temporary) registry = upstream / "implementations.json" original = json.dumps(self.registry) registry.write_text(original) matrices = [] + reported = [] def command(args, **kwargs): results = upstream / "results" if args[:2] == ["bash", "run-interop-tests.sh"]: - pairs = runner.expected_pairs(json.loads(registry.read_text())) + plan = json.loads(registry.read_text()) result = results / str(len(matrices)) result.mkdir(parents=True) runs = [ - {"client": client, "relay": relay, "mode": "docker", "status": status(client, relay)} - for client, relay in pairs + {"client": client, "relay": relay, "mode": mode, "status": status(client, relay, mode)} + for client, relay, mode in runner.expected_runs(plan) ] (result / "summary.json").write_text(json.dumps({"runs": runs})) - matrices.append(pairs) + matrices.append(pairs(plan)) + reported.extend(runs) failed = any(run["status"] != "pass" for run in runs) return subprocess.CompletedProcess(args, int(failed)) + if args[:2] == ["docker", "pull"]: + return subprocess.CompletedProcess(args, int(args[2] in unpublished)) if args[:2] == ["bash", "generate-report.sh"]: (results / "index.html").write_text("") (results / "nightly" / "report.html").write_text("") return subprocess.CompletedProcess(args, 0) failure = None - with patch.object(runner.subprocess, "run", side_effect=command): + self.matrices = matrices + with patch.object(runner.subprocess, "run", side_effect=command), patch.object(runner.time, "sleep"): try: runner.run(upstream, upstream / "report") except SystemExit as error: failure = error - self.assertEqual(len(matrices), 2) self.assertEqual(registry.read_text(), original) runs = json.loads((upstream / "report" / "nightly" / "summary.json").read_text())["runs"] - self.assertEqual(len(runs), sum(len(pairs) for pairs in matrices)) - self.assertIn("local-adapter", (upstream / "report" / "index.html").read_text()) + self.assertEqual(runs, reported) + self.index = (upstream / "report" / "index.html").read_text() + self.assertIn("local-adapter", self.index) return failure def test_failed_required_pair_reports_every_matrix_then_fails(self): required = ("moq-dev-js", "moq-dev-rs") - failure = self.run_matrices(lambda client, relay: "timeout" if (client, relay) == required else "pass") + failure = self.run_matrices(lambda client, relay, mode: "timeout" if (client, relay) == required else "pass") + self.assertEqual(len(self.matrices), 2) self.assertIn(str(required), str(failure)) def test_failed_optional_pairs_are_reported_but_pass(self): - def status(client, relay): + def status(client, relay, mode): return "pass" if (client, relay) in runner.REQUIRED else "fail" with patch("builtins.print") as report: failure = self.run_matrices(status) self.assertIsNone(failure) - self.assertIn("('new', 'moq-dev-rs')", report.call_args.args[0]) + self.assertIn("('new', 'moq-dev-rs', 'docker')", report.call_args.args[0]) + + def test_unpublished_third_party_image_drops_only_its_pairs(self): + with patch("builtins.print") as report: + failure = self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/client"}) + self.assertIsNone(failure) + self.assertEqual(self.matrices[0], runner.REQUIRED) + self.assertEqual(self.matrices[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")}) + self.assertIn("docker.io/new/client", report.call_args.args[0]) + self.assertIn("docker.io/new/client (image unavailable)", self.index) + + def test_unpublished_last_third_party_relay_skips_the_optional_matrix(self): + with patch("builtins.print"): + failure = self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/relay"}) + self.assertIsNone(failure) + self.assertEqual(self.matrices, [runner.REQUIRED | {("new", "moq-dev-rs")}]) + + def test_matrix_the_registry_leaves_empty_fails_despite_unrelated_missing_image(self): + del self.registry["implementations"]["new"]["roles"]["relay"] + with patch("builtins.print"), self.assertRaisesRegex(ValueError, "matrix 1 has no compatible pairs"): + self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/client"}) + + def test_unpublished_own_client_image_fails_before_any_matrix(self): + def command(args, **kwargs): + self.assertNotEqual(args[:2], ["bash", "run-interop-tests.sh"]) + return subprocess.CompletedProcess(args, int(args == ["docker", "pull", "ghcr.io/ours/js"])) + + with tempfile.TemporaryDirectory() as temporary: + upstream = Path(temporary) + (upstream / "implementations.json").write_text(json.dumps(self.registry)) + with ( + patch.object(runner.subprocess, "run", side_effect=command), + patch.object(runner.time, "sleep"), + self.assertRaises(SystemExit) as failure, + ): + runner.run(upstream, upstream / "report") + self.assertIn("ghcr.io/ours/js", str(failure.exception)) def test_required_pair_the_registry_drops_fails(self): self.registry["implementations"]["moq-dev-js"]["draft_versions"] = ["draft-22"] self.registry["implementations"]["new"]["draft_versions"] = ["draft-18", "draft-22"] - self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay: "pass"))) + self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay, mode: "pass"))) def test_drafts_without_overlap_have_no_pair(self): self.registry["implementations"]["new"]["draft_versions"] = ["draft-22"] plans, _ = runner.matrices(self.registry) - self.assertNotIn(("new", "moq-dev-rs"), runner.expected_pairs(plans[0])) - self.assertFalse(runner.expected_pairs(plans[1])) + self.assertNotIn(("new", "moq-dev-rs"), pairs(plans[0])) + self.assertFalse(pairs(plans[1])) if __name__ == "__main__":