Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ test:
python3 sh/size/release.test.py
python3 sh/size/nightly.test.py
python3 bench/compare.test.py
bash sh/local.sh python3 bench/build.test.py
python3 bench/summary.test.py
python3 bench/profile.test.py
python3 bench/fanout.test.py
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ Local nextest runs default to at most four test workers; set
`NEXTEST_TEST_THREADS` to override it. Bun workspace scripts run sequentially
locally, preserving dependency order, while CI keeps its workspace parallelism.
Selection and an empty JS selection do not wait for admission. For a build
outside these entrypoints, use `quest gate COMMAND [ARGS...]`; gate thresholds
outside these entrypoints, use `bash sh/quest.sh gate COMMAND [ARGS...]`; gate thresholds
come from the host and have no configuration.

Run `just ci` before pushing. Ready PRs into `main` also run Check as a signal; only GitHub's merge queue gates landing. `just ci` runs what the merge queue runs, the same ground as `just check`, plus two settings: `CI_STRICT=1`, which turns a missing tool into an error instead of a skipped check, and `NEXTEST_PROFILE=ci`, which allows a longer hang timeout.
Expand Down
3 changes: 2 additions & 1 deletion bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -902,7 +902,8 @@ nix develop --command just gh benchmark-report gate target/bench/nightly/rust-be

Each run's evidence directory keeps both builds' estimates (`criterion` and
`criterion-base`), the confirmation rounds under `confirm/`, and the run's
`paired.json`.
`paired.json`. The base build streams its diagnostics and retains them in
`base-build.log`, including when it fails.

### CodSpeed

Expand Down
101 changes: 101 additions & 0 deletions bench/build.test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Exercise paired builds with real Cargo diagnostics and target discovery."""

import json
import os
import shutil
import subprocess
import tempfile
import unittest
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]


class Builds(unittest.TestCase):
def test_base_builds_report_errors_and_distinguish_new_targets(self):
(ROOT / ".tmp").mkdir(exist_ok=True)
for base in ("broken", "existing", "new", "log-failure"):
with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory:
repo = Path(directory) / "repo"
repo.mkdir()
shutil.copytree(ROOT / "bench", repo / "bench", ignore=shutil.ignore_patterns("__pycache__"))
(repo / ".cargo").mkdir()
(repo / ".cargo/config.toml").touch()
(repo / ".gitignore").write_text(".tmp/\ntarget/\n")
(repo / "Cargo.toml").write_text('[workspace]\nmembers=["rs/moq-relay"]\nresolver="3"\n')
package = repo / "rs/moq-relay"
(package / "src").mkdir(parents=True)
(package / "benches").mkdir()
(package / "src/lib.rs").touch()
manifest = '[package]\nname="moq-relay"\nversion="0.0.0"\nedition="2024"\n'
target = '[[bench]]\nname="catalog"\nharness=false\n'
(package / "Cargo.toml").write_text(manifest + (target if base != "new" else ""))
bench = package / "benches/catalog.rs"
if base != "new":
bench.write_text(
'compile_error!("base catalog source failed");\n'
if base == "broken"
else 'fn main() { println!("base catalog ran"); }\n'
)
env = dict(os.environ, CARGO_TARGET_DIR=str(repo / "target"), BENCH_BASE="HEAD")
for key in ("BENCH_ARTIFACTS", "BENCH_RUNS", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER"):
env.pop(key, None)

def run(*args):
result = subprocess.run(args, cwd=repo, env=env, text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
return result

run("cargo", "generate-lockfile")
run("git", "init", "-q", "-b", "main")
run("git", "add", ".")
run(
"git",
"-c",
"user.name=Test",
"-c",
"user.email=test@example.com",
"-c",
"commit.gpgsign=false",
"commit",
"-qm",
"base",
)
(package / "Cargo.toml").write_text(manifest + target)
bench.write_text('fn main() { println!("current catalog ran"); }\n')
if base == "log-failure":
tools = repo / ".tmp/tools"
tools.mkdir(parents=True)
tee = tools / "tee"
tee.write_text(
"#!/usr/bin/env bash\n"
"if [[ $1 == */base-build.log ]]; then cat; exit 1; fi\n"
f'exec {shutil.which("tee")} "$@"\n'
)
tee.chmod(0o755)
env["PATH"] = str(tools) + os.pathsep + env["PATH"]
result = run("bash", "bench/run.sh", "--criterion", "-p", "moq-relay", "--bench", "catalog")
(evidence,) = (repo / ".tmp/bench").glob("run.*")
log = (evidence / "base-build.log").read_text() if base != "log-failure" else ""
report = json.loads((evidence / "paired.json").read_text())
self.assertIn("current catalog ran", result.stdout)
self.assertIn(str(evidence / "base-build.log"), result.stdout)
if base == "broken":
self.assertIn("base catalog source failed", result.stderr)
self.assertIn("base catalog source failed", log)
self.assertEqual(
report["unavailable"], [{"target": "moq-relay/catalog", "reason": "base build failed"}]
)
self.assertEqual((evidence / "criterion-base.targets").read_text(), "")
else:
self.assertEqual(report["unavailable"], [])
self.assertEqual("base catalog ran" in result.stdout, base in ("existing", "log-failure"))
if base == "log-failure":
self.assertIn("Failed to write base build log:", result.stderr)
self.assertNotEqual((evidence / "criterion-base.targets").read_text(), "")
self.assertEqual(len(run("git", "worktree", "list", "--porcelain").stdout.split("worktree ")) - 1, 1)


if __name__ == "__main__":
unittest.main()
8 changes: 7 additions & 1 deletion bench/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,13 @@ measure_criterion() {
mkdir -p "$RUN/base-context"
(cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" comparison-base)
printf 'Criterion paired with base %s\n' "$BASE"
if ! criterion_build_base "$base"; then
printf 'Base build log: %s/base-build.log\n' "$RUN"
local build_status=(0 0)
criterion_build_base "$base" 2>&1 | tee "$RUN/base-build.log" >&2 || build_status=("${PIPESTATUS[@]}")
if ((build_status[1] != 0)); then
printf 'Failed to write base build log: %s/base-build.log\n' "$RUN" >&2
fi
if ((build_status[0] != 0)); then
: >"$base"
for key in "${targets[@]}"; do unavailable "$key" "base build failed"; done
fi
Expand Down
8 changes: 8 additions & 0 deletions doc/setup/dev.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ module as skipped; `just ci` still requires the full flake check.
contains the module selection, with package selection delegated to the
language runners.

Repository gates and quest checks use the dev shell’s quest executable through
`sh/quest.sh`, reporting its path. Enter this checkout’s dev shell to use its pin. Use
`bash sh/quest.sh review --base origin/main` for a local review with that build,
or `bash sh/quest.sh gate COMMAND [ARGS...]` to admit a standalone build.
Outside the dev shell, explicit host toolchains (`ALLOW_HOST_TOOLCHAIN=1`, or Windows without Nix)
report the selected quest executable and version alongside the repository pin;
their build identity is unverified.

| What you changed or need to measure | Start here |
| --- | --- |
| Ordinary source changes | `just check`; [Rust instructions](https://github.com/moq-dev/moq/blob/main/rs/AGENTS.md) and [TypeScript instructions](https://github.com/moq-dev/moq/blob/main/js/AGENTS.md) describe focused test commands and conventions |
Expand Down
6 changes: 6 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -679,6 +679,9 @@
export PATH="$(dirname "$shim"):$PATH"
fi

# Skills also invoke quest directly, so keep it ahead of host installs.
export PATH="${quest-cli}/bin:$PATH"

# Each nested shell (direnv, then `nix develop`, then another
# inside a gate) prepends this shell's ~160 store paths again,
# about 10 KB. mbx copies PATH into two variables at ~3.6 times
Expand All @@ -693,6 +696,9 @@
# fix`. IN_NIX_SHELL would also pass inside another project's shell.
IN_MOQ_DEV_SHELL = "1";

# Local launchers need the exact build without evaluating the flake again.
MOQ_QUEST = "${quest-cli}/bin/quest";

PLAYWRIGHT_BROWSERS_PATH = "${pkgs.playwright-driver.browsers}";
PLAYWRIGHT_VERSION = pkgs.playwright-driver.version;
PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS = "true";
Expand Down
1 change: 0 additions & 1 deletion quest/m0/ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,6 @@ The historical samples locate candidates; they are not controlled benchmarks.
- [Unify dependency features](/quest/m0/ci/hakari.md) - align the FFI builds' features and record the hosted platform jobs' hits once the workspace-hack lands
- [Drop main from the publish Environment](/quest/m0/ci/publish-release-only.md) - the maintainer narrows `publish` to `release` and release tags once no job naming it runs as `main`
- [Binary release publishers run on their tag](/quest/m0/ci/release-tag-publishers.md) - brew, Go and Swift releases stop running through `workflow_run` on main
- [The bench comparison builds moq-gst's catalog bench on its base](/quest/m0/ci/bench-base-build.md) - A paired bench comparison that selects `rust/gst/moq-gst/catalog` against `origin/main` builds the base side and reports a comparison, instead of marking it unavailable with "base build failed"
- [Measure the Linux cache effect of native media exclusions](/quest/m0/ci/hakari-followups.md) - scheduled Linux jobs measure cache reuse after unrelated tests stop compiling PipeWire and libclang
- [A release merge opens no stale release PR](/quest/m0/ci/release-pr-after-release.md) - `just rs release` publishes before it proposes, so a merged release isn't re-proposed
- [Build two queue entries at once if it helps](/quest/m0/ci/queue-concurrency.md) - a measured trial of `maximumEntriesToBuild: 2` against the serial lane's burst latency
44 changes: 0 additions & 44 deletions quest/m0/ci/bench-base-build.md

This file was deleted.

11 changes: 9 additions & 2 deletions sh/dispatch-query.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def setUp(self):
self.root = Path(self.temp.name)
self.repo = self.root / "repo"
(self.repo / "sh").mkdir(parents=True)
for script in ("dispatch.sh", "changed.sh", "local.sh"):
for script in ("dispatch.sh", "changed.sh", "local.sh", "quest.sh"):
shutil.copy(ROOT / "sh" / script, self.repo / "sh" / script)
# The stub belongs in the base so it does not widen every test's diff.
(self.repo / "sh/kvm.sh").write_text("#!/bin/sh\nexit 0\n")
Expand Down Expand Up @@ -84,6 +84,11 @@ def test_quest_only_has_no_tests_but_keeps_repository_lints(self):
self.change("quest/m1/example.md")
self.expect(True, rust=False)

def test_quest_check_uses_the_checkout_pin(self):
self.change("quest/m1/example.md")
calls = self.run_ci()
self.assertIn("pinned quest check", calls)

def test_biome_checks_files_outside_the_js_impact_map(self):
for path in ("cpp/obs/buildspec.json", "sh/gh/setup.test.ts", ".github/ci-selection-audit.json"):
with self.subTest(path=path):
Expand Down Expand Up @@ -281,7 +286,9 @@ def stub_ci(self, just=""):
nix = self.bin / "nix"
nix.write_text('#!/usr/bin/env bash\nprintf "nix %s\\n" "$*" >>"$CALLS"\n')
nix.chmod(0o755)
self.env.update(IN_MOQ_DEV_SHELL="1", CALLS=str(self.root / "calls"))
quest = self.bin / "quest"
quest.write_text('#!/usr/bin/env bash\nprintf "pinned quest %s\\n" "$*" >>"$CALLS"\n')
self.env.update(IN_MOQ_DEV_SHELL="1", MOQ_QUEST=str(quest), CALLS=str(self.root / "calls"))
(self.root / "calls").unlink(missing_ok=True)
for script in ("markdown.sh", "shell.sh", "large-files.sh", "text.sh"):
path = self.repo / "sh" / script
Expand Down
4 changes: 2 additions & 2 deletions sh/dispatch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ module moq_sh '^infra/moq-sh/' \
tools='dash'
# Quest documents form one graph, so any change validates the whole tree,
# as does a bump of the flake-pinned validator.
module quest '^(quest/|flake\.lock$)' \
module quest '^(quest/|flake\.lock$|sh/quest\.sh$)' \
tools='quest'
# `obs check` compares the OBS pinned in buildspec.json, flake.nix, and
# nixpkgs, and the last moves on a flake.lock bump alone.
Expand Down Expand Up @@ -337,7 +337,7 @@ step() {
python3 bench/compare.test.py
bash sh/local.sh cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring
;;
check:quest) quest check ;;
check:quest) bash sh/quest.sh check ;;
check:drill) just test drill-sensitivity --apply-only ;;
check:obs_compile) just obs compile ;;
check:flake)
Expand Down
8 changes: 7 additions & 1 deletion sh/js/workspace.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ def plan(self, paths):
def test_dispatcher_passes_only_js_impact_paths(self):
sh = self.root / "sh"
shutil.copy(selector.ROOT / "sh/dispatch.sh", sh / "dispatch.sh")
shutil.copy(selector.ROOT / "sh/quest.sh", sh / "quest.sh")
changed = sh / "changed.sh"
changed.write_text("#!/bin/sh\nprintf '%s\\n' js/auth/src/token.ts demo/pub/worker.ts quest/unrelated.md\n")
changed.chmod(0o755)
Expand All @@ -75,7 +76,12 @@ def test_dispatcher_passes_only_js_impact_paths(self):
GITHUB_BASE_REF="missing",
GITHUB_STEP_SUMMARY=str(summary),
):
env = dispatch_env(PATH=f"{tools}:{os.environ['PATH']}", ALLOW_HOST_TOOLCHAIN="1", SELECT_CALLS=str(calls))
env = dispatch_env(
PATH=f"{tools}:{os.environ['PATH']}",
ALLOW_HOST_TOOLCHAIN="1",
MOQ_QUEST=str(tools / "quest"),
SELECT_CALLS=str(calls),
)
for script in ("markdown.sh", "shell.sh", "large-files.sh", "text.sh"):
path = sh / script
path.write_text("#!/bin/sh\nexit 0\n")
Expand Down
2 changes: 1 addition & 1 deletion sh/local.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ fi
if [[ "${QUEST_GATE_ADMITTED:-}" == 1 ]]; then
exec bun --no-orphans "${BASH_SOURCE[0]%/*}/local.ts" "$@"
fi
exec quest gate bun --no-orphans "${BASH_SOURCE[0]%/*}/local.ts" "$@"
exec bash "${BASH_SOURCE[0]%/*}/quest.sh" gate bun --no-orphans "${BASH_SOURCE[0]%/*}/local.ts" "$@"
Loading
Loading