Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 45 additions & 26 deletions test/smoke/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
REQUIRED = {(client, "moq-dev-rs") for client in OWN}


def matrices(registry):
"""Keep published images and partition the pairs around our relay and clients."""
def matrices(registry, missing=frozenset()):
"""Keep the published images not in `missing` and partition the pairs around our relay and clients."""
registry = copy.deepcopy(registry)
implementations = registry["implementations"]
relay = implementations["moq-dev-rs"]["roles"]["relay"]["docker"]
Expand All @@ -33,16 +33,14 @@ def matrices(registry):
continue
image = docker["image"]
# Local adapter tags need a build even when their base is published.
local = bool(docker.get("build")) and "/" not in image
inventory.append(
{
"implementation": name,
"role": role,
"image": image,
"status": "not run: local build only" if local else "selected",
}
)
if local:
if docker.get("build") and "/" not in image:
status = "not run: local build only"
elif image in missing:
status = "not run: image unavailable"
else:
status = "selected"
inventory.append({"implementation": name, "role": role, "image": image, "status": status})
if status != "selected":
del implementation["roles"][role]["docker"]
if any("docker" not in implementations[name]["roles"]["client"] for name in OWN):
raise ValueError("the registry must provide both moq-dev client images")
Expand All @@ -59,44 +57,61 @@ def matrices(registry):
return (to_relay, from_clients), inventory


def expected_pairs(registry):
def modes(docker):
"""The runner's modes for a docker relay: one per declared endpoint, else plain docker."""
return [f"docker-{endpoint['transport']}" for endpoint in docker.get("endpoints", [])] or ["docker"]


def expected_runs(registry):
"""Every (client, relay, mode) run the runner should report for this registry."""
implementations = registry["implementations"]
return {
(client, relay)
(client, relay, mode)
for client, client_info in implementations.items()
if client_info.get("roles", {}).get("client", {}).get("docker")
for relay, relay_info in implementations.items()
if relay_info.get("roles", {}).get("relay", {}).get("docker")
and set(client_info["draft_versions"]) & set(relay_info["draft_versions"])
for mode in modes(relay_info["roles"]["relay"]["docker"])
}


def validate_summary(summary, expected):
runs = summary["runs"]
actual = {(run["client"], run["relay"]) for run in runs}
actual = {(run["client"], run["relay"], run["mode"]) for run in runs}
if actual != expected or len(runs) != len(expected):
raise ValueError(f"incomplete runner coverage: expected {sorted(expected)}, got {sorted(actual)}")
if any(run["mode"] != "docker" or run["status"] not in ("pass", "fail", "timeout") for run in runs):
raise ValueError("unexpected mode, skipped image, or unknown verdict in runner summary")
if any(run["status"] not in ("pass", "fail", "timeout") for run in runs):
raise ValueError("skipped image or unknown verdict in runner summary")


def run(upstream, output):
if output.exists():
raise ValueError(f"report destination already exists: {output}")
registry_file = upstream / "implementations.json"
original = registry_file.read_text()
plans, inventory = matrices(json.loads(original))
registry = json.loads(original)
for command in (["docker", "info"], ["docker", "compose", "version"]):
subprocess.run(command, check=True)
unfiltered, inventory = matrices(registry)
own = {registry["implementations"][name]["roles"]["client"]["docker"]["image"] for name in OWN}
missing = set()
images = {entry["image"] for entry in inventory if entry["status"] == "selected"}
for image in sorted(images - {RELAY_IMAGE}):
# A registry blip would otherwise abort both matrices before any report.
for attempt in range(1, 4):
if not subprocess.run(["docker", "pull", image]).returncode:
break
if attempt == 3:
if attempt < 3:
time.sleep(attempt * 2)
elif image in own:
raise SystemExit(f"failed to pull {image}")
time.sleep(attempt * 2)
else:
# A third party's unpublished image only removes its optional pairs, listed under Not run.
missing.add(image)
if missing:
print(f"::warning::images unavailable, their pairs not run: {sorted(missing)}")
plans, inventory = matrices(registry, missing)
subprocess.run(["docker", "image", "inspect", RELAY_IMAGE], check=True)
subprocess.run(["make", "certs"], cwd=upstream, check=True)
results = upstream / "results"
Expand All @@ -107,8 +122,11 @@ def run(upstream, output):
try:
for index, plan in enumerate(plans):
registry_file.write_text(json.dumps(plan))
expected = expected_pairs(plan)
expected = expected_runs(plan)
if not expected:
# Unpublished images can empty the optional matrix; the registry alone must not.
if expected_runs(unfiltered[index]):
continue
Comment thread
kixelated marked this conversation as resolved.
Resolved
raise ValueError(f"runner matrix {index} has no compatible pairs")
# The runner exits nonzero on any red pair; the validated summary decides instead.
subprocess.run(["bash", "run-interop-tests.sh", "--docker-only"], cwd=upstream)
Expand Down Expand Up @@ -137,7 +155,7 @@ def run(upstream, output):
if skipped:
rows = "".join(
f"<li>{html.escape(entry['implementation'])} {entry['role']}: "
f"{html.escape(entry['image'])} (local build only)</li>"
f"{html.escape(entry['image'])} ({html.escape(entry['status'].removeprefix('not run: '))})</li>"
for entry in skipped
)
index_file = results / "index.html"
Expand All @@ -150,12 +168,13 @@ def run(upstream, output):
)
# Fail only after the report shows every pair. A required pair the registry
# no longer pairs fails too, rather than passing unchecked.
status = {(run["client"], run["relay"]): run["status"] for run in combined["runs"]}
optional = sorted(pair for pair, verdict in status.items() if verdict != "pass" and pair not in REQUIRED)
red = {(run["client"], run["relay"], run["mode"]) for run in combined["runs"] if run["status"] != "pass"}
optional = sorted(run for run in red if run[:2] not in REQUIRED)
if optional:
# A workflow annotation, so a green job still shows them.
print(f"::warning::optional pairs not passing (reported only): {optional}")
failed = sorted(pair for pair in REQUIRED if status.get(pair) != "pass")
print(f"::warning::optional runs not passing (reported only): {optional}")
ran = {(run["client"], run["relay"]) for run in combined["runs"]}
failed = sorted(pair for pair in REQUIRED if pair not in ran or any(run[:2] == pair for run in red))
if failed:
raise SystemExit(f"required pairs failed: {failed}; see the report")

Expand Down
113 changes: 89 additions & 24 deletions test/smoke/runner.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ def implementation(**roles):
}


def pairs(plan):
return {(client, relay) for client, relay, _ in runner.expected_runs(plan)}


class Runner(unittest.TestCase):
def setUp(self):
self.registry = {
Expand All @@ -35,10 +39,10 @@ def setUp(self):

def test_new_published_implementations_join_both_directions_once(self):
plans, inventory = runner.matrices(self.registry)
pairs = [runner.expected_pairs(plan) for plan in plans]
self.assertEqual(pairs[0], {("moq-dev-rs", "moq-dev-rs"), ("moq-dev-js", "moq-dev-rs"), ("new", "moq-dev-rs")})
self.assertEqual(pairs[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")})
self.assertFalse(pairs[0] & pairs[1])
matrix = [pairs(plan) for plan in plans]
self.assertEqual(matrix[0], {("moq-dev-rs", "moq-dev-rs"), ("moq-dev-js", "moq-dev-rs"), ("new", "moq-dev-rs")})
self.assertEqual(matrix[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")})
self.assertFalse(matrix[0] & matrix[1])
self.assertEqual(
next(entry["status"] for entry in inventory if entry["implementation"] == "adapter"),
"not run: local build only",
Expand All @@ -50,80 +54,141 @@ def test_new_published_implementations_join_both_directions_once(self):
def test_published_image_with_source_recipe_is_still_selected(self):
self.registry["implementations"]["new"]["roles"]["relay"]["docker"]["build"] = {"context": "."}
plans, _ = runner.matrices(self.registry)
self.assertIn(("moq-dev-rs", "new"), runner.expected_pairs(plans[1]))
self.assertIn(("moq-dev-rs", "new"), pairs(plans[1]))

def test_missing_skipped_duplicate_or_unexpected_cells_fail(self):
cell = {"client": "a", "relay": "b", "mode": "docker", "status": "fail"}
runner.validate_summary({"runs": [cell]}, {("a", "b")})
for cells in ([], [cell, cell], [dict(cell, status="skip")], [dict(cell, relay="c")]):
runner.validate_summary({"runs": [cell]}, {("a", "b", "docker")})
wrong = [dict(cell, status="skip"), dict(cell, relay="c"), dict(cell, mode="docker-quic")]
for cells in ([], [cell, cell], *([bad] for bad in wrong)):
with self.subTest(cells=cells), self.assertRaises(ValueError):
runner.validate_summary({"runs": cells}, {("a", "b")})
runner.validate_summary({"runs": cells}, {("a", "b", "docker")})

def test_relay_endpoints_each_run_and_gate_their_pair(self):
self.registry["implementations"]["new"]["roles"]["relay"]["docker"]["endpoints"] = [
{"transport": "quic", "url": "moqt://relay:4443"},
{"transport": "webtransport", "url": "https://relay:4443"},
]
plans, _ = runner.matrices(self.registry)
self.assertEqual(
{run for run in runner.expected_runs(plans[1]) if run[0] == "moq-dev-rs"},
{("moq-dev-rs", "new", "docker-quic"), ("moq-dev-rs", "new", "docker-webtransport")},
)
with patch("builtins.print") as report:
failure = self.run_matrices(lambda client, relay, mode: "fail" if mode == "docker-quic" else "pass")
self.assertIsNone(failure)
self.assertIn("('moq-dev-js', 'new', 'docker-quic')", report.call_args.args[0])

def run_matrices(self, status, unpublished=frozenset()):
"""Run both matrices against a fake runner that gives each pair `status(client, relay)`.

def run_matrices(self, status):
"""Run both matrices against a fake runner that gives each pair `status(client, relay)`."""
Pulling an image in `unpublished` always fails.
"""
with tempfile.TemporaryDirectory() as temporary:
upstream = Path(temporary)
registry = upstream / "implementations.json"
original = json.dumps(self.registry)
registry.write_text(original)
matrices = []
reported = []

def command(args, **kwargs):
results = upstream / "results"
if args[:2] == ["bash", "run-interop-tests.sh"]:
pairs = runner.expected_pairs(json.loads(registry.read_text()))
plan = json.loads(registry.read_text())
result = results / str(len(matrices))
result.mkdir(parents=True)
runs = [
{"client": client, "relay": relay, "mode": "docker", "status": status(client, relay)}
for client, relay in pairs
{"client": client, "relay": relay, "mode": mode, "status": status(client, relay, mode)}
for client, relay, mode in runner.expected_runs(plan)
]
(result / "summary.json").write_text(json.dumps({"runs": runs}))
matrices.append(pairs)
matrices.append(pairs(plan))
reported.extend(runs)
failed = any(run["status"] != "pass" for run in runs)
return subprocess.CompletedProcess(args, int(failed))
if args[:2] == ["docker", "pull"]:
return subprocess.CompletedProcess(args, int(args[2] in unpublished))
if args[:2] == ["bash", "generate-report.sh"]:
(results / "index.html").write_text("<body></body>")
(results / "nightly" / "report.html").write_text("")
return subprocess.CompletedProcess(args, 0)

failure = None
with patch.object(runner.subprocess, "run", side_effect=command):
self.matrices = matrices
with patch.object(runner.subprocess, "run", side_effect=command), patch.object(runner.time, "sleep"):
try:
runner.run(upstream, upstream / "report")
except SystemExit as error:
failure = error
self.assertEqual(len(matrices), 2)
self.assertEqual(registry.read_text(), original)
runs = json.loads((upstream / "report" / "nightly" / "summary.json").read_text())["runs"]
self.assertEqual(len(runs), sum(len(pairs) for pairs in matrices))
self.assertIn("local-adapter", (upstream / "report" / "index.html").read_text())
self.assertEqual(runs, reported)
self.index = (upstream / "report" / "index.html").read_text()
self.assertIn("local-adapter", self.index)
return failure

def test_failed_required_pair_reports_every_matrix_then_fails(self):
required = ("moq-dev-js", "moq-dev-rs")
failure = self.run_matrices(lambda client, relay: "timeout" if (client, relay) == required else "pass")
failure = self.run_matrices(lambda client, relay, mode: "timeout" if (client, relay) == required else "pass")
self.assertEqual(len(self.matrices), 2)
self.assertIn(str(required), str(failure))

def test_failed_optional_pairs_are_reported_but_pass(self):
def status(client, relay):
def status(client, relay, mode):
return "pass" if (client, relay) in runner.REQUIRED else "fail"

with patch("builtins.print") as report:
failure = self.run_matrices(status)
self.assertIsNone(failure)
self.assertIn("('new', 'moq-dev-rs')", report.call_args.args[0])
self.assertIn("('new', 'moq-dev-rs', 'docker')", report.call_args.args[0])

def test_unpublished_third_party_image_drops_only_its_pairs(self):
with patch("builtins.print") as report:
failure = self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/client"})
self.assertIsNone(failure)
self.assertEqual(self.matrices[0], runner.REQUIRED)
self.assertEqual(self.matrices[1], {("moq-dev-rs", "new"), ("moq-dev-js", "new")})
self.assertIn("docker.io/new/client", report.call_args.args[0])
self.assertIn("docker.io/new/client (image unavailable)", self.index)

def test_unpublished_last_third_party_relay_skips_the_optional_matrix(self):
with patch("builtins.print"):
failure = self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/relay"})
self.assertIsNone(failure)
self.assertEqual(self.matrices, [runner.REQUIRED | {("new", "moq-dev-rs")}])

def test_matrix_the_registry_leaves_empty_fails_despite_unrelated_missing_image(self):
del self.registry["implementations"]["new"]["roles"]["relay"]
with patch("builtins.print"), self.assertRaisesRegex(ValueError, "matrix 1 has no compatible pairs"):
self.run_matrices(lambda client, relay, mode: "pass", unpublished={"docker.io/new/client"})

def test_unpublished_own_client_image_fails_before_any_matrix(self):
def command(args, **kwargs):
self.assertNotEqual(args[:2], ["bash", "run-interop-tests.sh"])
return subprocess.CompletedProcess(args, int(args == ["docker", "pull", "ghcr.io/ours/js"]))

with tempfile.TemporaryDirectory() as temporary:
upstream = Path(temporary)
(upstream / "implementations.json").write_text(json.dumps(self.registry))
with (
patch.object(runner.subprocess, "run", side_effect=command),
patch.object(runner.time, "sleep"),
self.assertRaises(SystemExit) as failure,
):
runner.run(upstream, upstream / "report")
self.assertIn("ghcr.io/ours/js", str(failure.exception))

def test_required_pair_the_registry_drops_fails(self):
self.registry["implementations"]["moq-dev-js"]["draft_versions"] = ["draft-22"]
self.registry["implementations"]["new"]["draft_versions"] = ["draft-18", "draft-22"]
self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay: "pass")))
self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay, mode: "pass")))

def test_drafts_without_overlap_have_no_pair(self):
self.registry["implementations"]["new"]["draft_versions"] = ["draft-22"]
plans, _ = runner.matrices(self.registry)
self.assertNotIn(("new", "moq-dev-rs"), runner.expected_pairs(plans[0]))
self.assertFalse(runner.expected_pairs(plans[1]))
self.assertNotIn(("new", "moq-dev-rs"), pairs(plans[0]))
self.assertFalse(pairs(plans[1]))


if __name__ == "__main__":
Expand Down