Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ runner-userns:
runner-vm:
bash sh/local.sh nix build --no-link .#checks.x86_64-linux.github-runner-vm .#checks.x86_64-linux.github-runner-vm-image .#checks.x86_64-linux.github-runner-vm-config --print-build-logs

# Lint GitHub Actions workflows, their security, their alert coverage, that every step runs a recipe, that every opt-in test has a caller, and that every benchmark test Nightly filters on still exists in the Rust sources.
# Lint GitHub Actions workflows, their security, their alert coverage, that every step runs a recipe, that every opt-in test has a caller, that every benchmark test Nightly filters on still exists in the Rust sources, and that the gate's Rust scope covers every file the Rust sources read outside rs/.
check:
python3 sh/version.py
actionlint
Expand All @@ -59,6 +59,7 @@ check:
python3 sh/gh/test-inventory.py
python3 bench/relay.test.py
python3 sh/gh/nightly.test.py NightlyTest.test_every_nightly_benchmark_test_exists
python3 sh/rs/select.test.py SelectTest.test_dispatch_scope_covers_rust_reads

# Repository tooling tests; the merge gate runs every ordinary test file whole when tooling changes.
test:
Expand Down
54 changes: 0 additions & 54 deletions quest/m0/ci/rust-test-reads.md

This file was deleted.

3 changes: 1 addition & 2 deletions rs/moq-audio/src/jitter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -588,8 +588,7 @@ pub(crate) mod tests {
/// implementation is graded against the other.
pub(crate) fn case(name: &str) -> Case {
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../doc/concept/audio-jitter")
.join(format!("{name}.json"));
.join(format!("../../doc/concept/audio-jitter/{name}.json"));
let json = std::fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display()));
serde_json::from_str(&json).unwrap()
}
Expand Down
5 changes: 2 additions & 3 deletions rs/moq-net/src/ietf/publisher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22345,8 +22345,7 @@ mod compression_tests {
use crate::lite::test_transport::{Log, SinkSession};
for (version, draft) in [(Version::Draft18, 18), (Version::Draft22, 22)] {
let input = serde_json::json!({"draft": draft});
let bytes: Vec<u8> =
serde_json::from_slice(&crate::test_interop::bun("ietf-compression.ts", input)).unwrap();
let bytes: Vec<u8> = serde_json::from_slice(&crate::test_interop::bun("ietf-compression", input)).unwrap();
let (parameters, consumed) = ietf::Parameters::decode_slice(&bytes, version).unwrap();
assert_eq!(consumed, bytes.len());
let negotiated = super::super::flate::from_setup(&parameters).unwrap();
Expand Down Expand Up @@ -22384,7 +22383,7 @@ mod compression_tests {
let written = log.writes.lock().unwrap().clone();
let input = serde_json::json!({"draft": draft, "response": response.to_vec(), "group": written});
let received: Vec<Vec<u8>> =
serde_json::from_slice(&crate::test_interop::bun("ietf-compression.ts", input)).unwrap();
serde_json::from_slice(&crate::test_interop::bun("ietf-compression", input)).unwrap();
assert_eq!(received, frames, "draft{draft}: plain JS fallback");
}
}
Expand Down
25 changes: 12 additions & 13 deletions rs/moq-net/src/test_interop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
use crate::coding::{Decoder, Encode, Encoder, Form, varint};
use crate::{ietf, lite};

// Runs a script with a JSON argument, returning its stdout.
// Runs `test/interop/<script>.ts` with a JSON argument, returning its stdout.
pub(crate) fn bun(script: &str, input: serde_json::Value) -> Vec<u8> {
let output = std::process::Command::new("bun")
.arg(format!("{}/../../test/interop/{script}", env!("CARGO_MANIFEST_DIR")))
.arg(format!("{}/../../test/interop/{script}.ts", env!("CARGO_MANIFEST_DIR")))
.arg(input.to_string())
.output()
.expect("Bun must be installed for the interop tests");
Expand All @@ -21,14 +21,14 @@ pub(crate) fn bun(script: &str, input: serde_json::Value) -> Vec<u8> {

pub(crate) fn fin(version: &str, started: bool, clean: bool, responses: Vec<u8>) -> Vec<u8> {
let input = serde_json::json!({ "version": version, "started": started, "clean": clean, "responses": responses });
serde_json::from_slice(&bun("bare-fin.ts", input)).expect("JS publisher returned response bytes")
serde_json::from_slice(&bun("bare-fin", input)).expect("JS publisher returned response bytes")
}

/// Feed a JS subscriber responses that end the track owing group 0, kill its session, and
/// return the JS publisher's encoding of the same responses.
pub(crate) fn owed_tail(version: &str, responses: Vec<u8>) -> Vec<u8> {
let input = serde_json::json!({ "version": version, "owed": true, "responses": responses });
serde_json::from_slice(&bun("bare-fin.ts", input)).expect("JS publisher returned response bytes")
serde_json::from_slice(&bun("bare-fin", input)).expect("JS publisher returned response bytes")
}

/// Rust updates keep the JS subscription alive, cover both unseen groups after narrowing,
Expand Down Expand Up @@ -109,7 +109,7 @@ fn ietf_range_updates_interop() {
"subscribe": subscribe,
"updates": updates,
});
let served: Served = serde_json::from_slice(&bun("ietf-range-updates.ts", input)).unwrap();
let served: Served = serde_json::from_slice(&bun("ietf-range-updates", input)).unwrap();
assert_eq!(served.responses.len(), 3);
for (index, bytes) in served.responses.iter().enumerate() {
let mut r = Decoder::new(bytes, version.into());
Expand Down Expand Up @@ -246,8 +246,7 @@ fn varint_interop() {
"quic": values.iter().map(|&v| encode(v, quic)).collect::<Vec<_>>(),
"leadingOnes": values.iter().map(|&v| encode(v, leading_ones)).collect::<Vec<_>>(),
});
let output: serde_json::Value =
serde_json::from_slice(&bun("varint.ts", input)).expect("JS returned its encodings");
let output: serde_json::Value = serde_json::from_slice(&bun("varint", input)).expect("JS returned its encodings");
let js = |format: &str| -> Vec<Vec<u8>> { serde_json::from_value(output[format].clone()).unwrap() };

for ((&value, js_quic), js_leading) in values.iter().zip(js("quic")).zip(js("leadingOnes")) {
Expand Down Expand Up @@ -383,7 +382,7 @@ fn lite_varint_interop() {
"group": group,
"untimedGroup": untimed_group,
});
let echo: Echo = serde_json::from_slice(&bun("lite-varint.ts", input)).expect("JS returned its encodings");
let echo: Echo = serde_json::from_slice(&bun("lite-varint", input)).expect("JS returned its encodings");

let decoded: Vec<u64> = echo.values.iter().map(|v| v.parse().unwrap()).collect();
assert_eq!(decoded, values, "{version}: JS decoded different values");
Expand Down Expand Up @@ -469,7 +468,7 @@ fn ietf_datagram_interop() {

let input = serde_json::json!({ "draft": draft, "datagrams": encoded });
let echo: DatagramEcho =
serde_json::from_slice(&bun("ietf-datagram.ts", input)).expect("JS returned its encodings");
serde_json::from_slice(&bun("ietf-datagram", input)).expect("JS returned its encodings");

assert_eq!(echo.echoes, encoded, "{version}: JS re-encoded differently");
let (published, used) = ietf::ObjectDatagram::decode_slice(&echo.published, version).unwrap();
Expand Down Expand Up @@ -500,7 +499,7 @@ fn lite_auth_unsupported_interop() {
let mut writer = Encoder::new(&mut bytes, version.into());
grant.encode(&mut writer, version).unwrap();
refused.encode(&mut writer, version).unwrap();
let echo: Vec<u8> = serde_json::from_slice(&bun("auth-unsupported.ts", serde_json::json!(bytes))).unwrap();
let echo: Vec<u8> = serde_json::from_slice(&bun("auth-unsupported", serde_json::json!(bytes))).unwrap();
assert_eq!(echo, bytes);
let mut reader = Decoder::new(&echo, version.into());
assert_eq!(
Expand Down Expand Up @@ -531,7 +530,7 @@ fn lite_tracks_interop() {
.encode_bytes(version)
.unwrap();
let input = serde_json::json!({ "mode": "answer", "bytes": request.to_vec() });
let answer: Vec<u8> = serde_json::from_slice(&bun("tracks.ts", input)).expect("JS returned its listing");
let answer: Vec<u8> = serde_json::from_slice(&bun("tracks", input)).expect("JS returned its listing");
let mut r = Decoder::new(&answer, version.into());
let mut events = Vec::new();
while !r.is_empty() {
Expand Down Expand Up @@ -609,7 +608,7 @@ fn lite_tracks_interop() {
event.encode(&mut w, version).unwrap();
}
let input = serde_json::json!({ "mode": "echo", "bytes": bytes });
let echo: Vec<u8> = serde_json::from_slice(&bun("tracks.ts", input)).expect("JS echoed the events");
let echo: Vec<u8> = serde_json::from_slice(&bun("tracks", input)).expect("JS echoed the events");
assert_eq!(echo, bytes);
}

Expand Down Expand Up @@ -647,7 +646,7 @@ fn ietf_track_status_interop() {
"requests": [request(1, "room", true), request(3, "room", false), request(5, "absent", true)],
});
let responses: Vec<Vec<u8>> =
serde_json::from_slice(&bun("track-status.ts", input)).expect("JS returned its responses");
serde_json::from_slice(&bun("track-status", input)).expect("JS returned its responses");
assert_eq!(responses.len(), 3, "{version}");

for (index, wanted) in [true, false].into_iter().enumerate() {
Expand Down
39 changes: 22 additions & 17 deletions rs/moq-relay/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1104,31 +1104,36 @@ max_streams = 64
/// Real relay examples, packaging and test fixtures use the same loader as an embedding CLI.
#[test]
fn repository_configs_load() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let manifest = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
// Only repository inputs belong here; installed clients and build output below
// test/ can contain unrelated TOML. Git's tracked list also includes new fixtures.
let tracked = std::process::Command::new("git")
.args(["ls-files", "-z", "--", "demo/relay", "packaging", "test"])
.current_dir(&root)
.output()
.expect("list repository fixtures");
assert!(tracked.status.success());
let tracked = String::from_utf8(tracked.stdout).unwrap();
for area in ["demo/relay", "packaging", "test"] {
// Each glob names its files from this crate, so sh/rs/reads.sh finds the read
// and a new fixture selects this test.
for area in [
"../../demo/relay/**/*.toml",
"../../packaging/**/*.toml",
"../../test/**/*.toml",
] {
let tracked = std::process::Command::new("git")
.args(["ls-files", "-z", "--", &format!(":(glob){area}")])
.current_dir(manifest)
.output()
.expect("list repository fixtures");
assert!(tracked.status.success());
let tracked = String::from_utf8(tracked.stdout).unwrap();
let mut checked = 0;
// test/mesh-replay/ holds mesh topologies and event scripts, not relay configs; the
// replay writes each relay's config at run time.
for name in tracked
.split('\0')
.filter(|name| name.starts_with(area) && !name.starts_with("test/mesh-replay/"))
.filter(|name| !name.is_empty() && !name.contains("/test/mesh-replay/"))
{
let path = root.join(name);
if path.extension().is_none_or(|ext| ext != "toml")
|| matches!(
path.file_name().and_then(|name| name.to_str()),
// `cases.toml` holds test/quic-interop's expected results.
Some("Cargo.toml" | "rust-toolchain.toml" | "bunfig.toml" | "cases.toml")
) {
let path = manifest.join(name);
if matches!(
path.file_name().and_then(|name| name.to_str()),
// `cases.toml` holds test/quic-interop's expected results.
Some("Cargo.toml" | "rust-toolchain.toml" | "bunfig.toml" | "cases.toml")
) {
continue;
}
let result = Config::parse_and_merge([std::ffi::OsString::from("moq-relay"), path.clone().into()]);
Expand Down
10 changes: 6 additions & 4 deletions sh/dispatch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -115,9 +115,11 @@ module py '^(py/|sh/py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/|rs/uniffi-bindgen
# sh/rs/select.sh widens to the whole workspace for inputs every crate shares.
# moq-cli builds the Rust interop clients as examples from outside its
# crate; sh/rs/select.test.py fails when a target's directory is missing.
# moq-relay's tests read its NixOS module, and libmoq's lint compiles the C
# doc samples.
module rs '^(\.cargo/|nix/modules/moq-relay\.nix$|doc/lib/c/|doc/lib/samples\.sh$|sh/local\.(sh|ts)$|rs/|test/interop/clients/rust/|sh/rs/|Cargo\.(toml|lock)$|clippy\.toml$|rust-toolchain\.toml$|\.config/nextest\.toml$)' \
# libmoq's lint compiles the C doc samples. The files Rust reads outside rs/,
# which sh/rs/reads.sh lists and select.test.py holds this scope to, come
# last: the relay's configs, unit, and NixOS module, the scripts the tests
# run, and the vectors the tests share with JS and the docs.
module rs '^(\.cargo/|doc/lib/c/|doc/lib/samples\.sh$|sh/local\.(sh|ts)$|rs/|test/interop/clients/rust/|sh/rs/|Cargo\.(toml|lock)$|clippy\.toml$|rust-toolchain\.toml$|\.config/nextest\.toml$|nix/modules/moq-relay\.nix$|(demo/relay|packaging|test)/.*\.toml$|packaging/moq-relay/moq-relay\.service$|test/interop/[^/]*\.ts$|test/(datagram-tracks/client|max-age/client|receive-credit/sender|relay-fill/relay|stream-limits/client)\.ts$|doc/concept/audio-jitter/.*\.json$|drafts/moq-e2ee-01-wip\.json$)' \
tools='cargo jq cc' compiles='check fix' fix
# nix/Cargo.nix records each crate's dependencies, features, and targets,
# read from the manifests, the lockfile, and the files cargo discovers targets
Expand Down Expand Up @@ -213,7 +215,7 @@ module markdown '' \
# Workflow lints and repository-wide policy, such as every opt-in test having a
# caller, so they gate every diff.
module gh '' \
tools='actionlint zizmor bun python3 uv cargo' lane=bun
tools='actionlint zizmor bun python3 uv cargo jq' lane=bun
# `just gh test`, the repository tooling tests, follow what they run and read,
# including the selectors under sh/, the release-plz configuration, the nfpm
# configs, the Cargo and Bun workspace manifests and lockfiles, and the flake
Expand Down
51 changes: 51 additions & 0 deletions sh/rs/reads.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
# Print each path outside its crate that a Rust source under rs/ reads, one
# `<crate dir> <glob> <file>:<line>` per line, naming the literal behind it. A glob's `*`, like a format interpolation such
# as `{name}`, stays within a path segment, `**` crosses them, and a plain path
# also covers what lies beneath it.
#
# A read is an `include_*!` or `#[path]` literal, relative to its file, or, in a
# file that mentions CARGO_MANIFEST_DIR, a literal that climbs out with `../`,
# relative to its crate. A `#[path]` to a `mod.rs` reads its directory, where its
# own modules live. The CARGO_MANIFEST_DIR rule spans the whole file rather than
# the statement that joins the literal, so a read built across statements is
# never missed; the cost is that any `../` string in such a file, such as a
# path-traversal test input, also counts, and the source column points at it. sh/rs/select.sh selects the reading crate from these, and
# sh/rs/select.test.py checks that sh/dispatch.sh's `rs` scope covers those
# outside rs/. The interop clients under test/ are moq-cli examples that only
# the interop harness runs, so their scripts stay out.
set -euo pipefail

cd "$(git rev-parse --show-toplevel)"

crates=$(git ls-files --cached --others --exclude-standard -- ':(glob)rs/**/Cargo.toml' | sed 's|/Cargo\.toml$||')
# grep exits 1 when no source climbs, which leaves nothing to print.
files=$(grep -rlE --include='*.rs' --exclude-dir=target '"(\{\}/|/)?\.\./' rs) || [[ $? == 1 ]]
[[ -n "$files" ]] || exit 0

# One argument per file; no path under rs/ holds whitespace.
# shellcheck disable=SC2086
jq -nrR --arg crates "$crates" '
def resolve: reduce split("/")[] as $s ([];
if $s == ".." then .[:-1] elif $s == "." or $s == "" then . else . + [$s] end) | join("/");
($crates | split("\n") | map(select(. != ""))) as $crates
| [inputs | {file: input_filename, line: .}]
| group_by(.file)[]
| .[0].file as $file
| (map(.line) | join("\n")) as $text
| ($file | split("/")[:-1] | join("/")) as $dir
| ([$crates[] | select(. as $c | $file | startswith($c + "/"))] | max_by(length)) as $crate
| ($text | test("CARGO_MANIFEST_DIR")) as $manifest
| $text
| match("(include(?:_str|_bytes)?!\\s*\\(\\s*|#\\[path\\s*=\\s*)?\"((?:\\{\\}/|/)?\\.\\./[^\"]*)\""; "g")
| "\($file):\($text[:.offset] | split("\n") | length)" as $source
| .captures as [$from, $name]
| if $from.string != null then "\($dir)/\($name.string)"
elif $manifest then "\($crate)/\($name.string | ltrimstr("{}"))"
Comment thread
kixelated marked this conversation as resolved.
Resolved
else empty end
| gsub("\\{[^}]*\\}"; "*")
| resolve
| if ($from.string // "") | startswith("#") then sub("/mod\\.rs$"; "") else . end
| select(startswith($crate + "/") | not)
| "\($crate) \(.) \($source)"
' $files | LC_ALL=C sort -u
Loading